{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations for Colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "IN_COLAB = \"google.colab\" in sys.modules\n",
    "\n",
    "if IN_COLAB:\n",
    "    !apt install python-opengl\n",
    "    !apt install ffmpeg\n",
    "    !apt install xvfb\n",
    "    !pip install pyvirtualdisplay\n",
    "    !pip install gym[all]\n",
    "    from pyvirtualdisplay import Display\n",
    "    \n",
    "    # Start virtual display\n",
    "    dis = Display(visible=0, size=(400, 400))\n",
    "    dis.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 08. Rainbow\n",
    "\n",
    "[M. Hessel et al., \"Rainbow: Combining Improvements in Deep Reinforcement Learning.\" arXiv preprint arXiv:1710.02298, 2017.](https://arxiv.org/pdf/1710.02298.pdf)\n",
    "\n",
    "We will integrate all the following seven components into a single integrated agent, which is called Rainbow!\n",
    "\n",
    "1. DQN\n",
    "2. Double DQN\n",
    "3. Prioritized Experience Replay\n",
    "4. Dueling Network\n",
    "5. Noisy Network\n",
    "6. Categorical DQN\n",
    "7. N-step Learning\n",
    "\n",
    "This method shows an impressive performance on the Atari 2600 benchmark, both in terms of data efficiency and final performance. \n",
    "\n",
    "![rainbow](https://user-images.githubusercontent.com/14961526/60591412-61748100-9dd9-11e9-84fb-076c7a61fbab.png)\n",
    "\n",
    "However, the integration is not so simple because some of components are not independent each other, so we will look into a number of points that people especailly feel confused.\n",
    "\n",
    "1. Noisy Network <-> Dueling Network\n",
    "2. Dueling Network <-> Categorical DQN\n",
    "3. Categorical DQN <-> Double DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import os\n",
    "import random\n",
    "from collections import deque\n",
    "from typing import Deque, Dict, List, Tuple\n",
    "\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from IPython.display import clear_output\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "\n",
    "# download segment tree module\n",
    "if IN_COLAB:\n",
    "    !wget https://raw.githubusercontent.com/curt-park/rainbow-is-all-you-need/master/segment_tree.py\n",
    "\n",
    "from segment_tree import MinSegmentTree, SumSegmentTree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replay buffer\n",
    "\n",
    "Same as the basic N-step buffer. \n",
    "\n",
    "(Please see *01.dqn.ipynb*, *07.n_step_learning.ipynb* for detailed description about the basic (n-step) replay buffer.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\"A simple numpy replay buffer.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        obs_dim: int, \n",
    "        size: int, \n",
    "        batch_size: int = 32, \n",
    "        n_step: int = 1, \n",
    "        gamma: float = 0.99\n",
    "    ):\n",
    "        self.obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
    "        self.next_obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
    "        self.acts_buf = np.zeros([size], dtype=np.float32)\n",
    "        self.rews_buf = np.zeros([size], dtype=np.float32)\n",
    "        self.done_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.max_size, self.batch_size = size, batch_size\n",
    "        self.ptr, self.size, = 0, 0\n",
    "        \n",
    "        # for N-step Learning\n",
    "        self.n_step_buffer = deque(maxlen=n_step)\n",
    "        self.n_step = n_step\n",
    "        self.gamma = gamma\n",
    "\n",
    "    def store(\n",
    "        self, \n",
    "        obs: np.ndarray, \n",
    "        act: np.ndarray, \n",
    "        rew: float, \n",
    "        next_obs: np.ndarray, \n",
    "        done: bool,\n",
    "    ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n",
    "        transition = (obs, act, rew, next_obs, done)\n",
    "        self.n_step_buffer.append(transition)\n",
    "\n",
    "        # single step transition is not ready\n",
    "        if len(self.n_step_buffer) < self.n_step:\n",
    "            return ()\n",
    "        \n",
    "        # make a n-step transition\n",
    "        rew, next_obs, done = self._get_n_step_info(\n",
    "            self.n_step_buffer, self.gamma\n",
    "        )\n",
    "        obs, act = self.n_step_buffer[0][:2]\n",
    "        \n",
    "        self.obs_buf[self.ptr] = obs\n",
    "        self.next_obs_buf[self.ptr] = next_obs\n",
    "        self.acts_buf[self.ptr] = act\n",
    "        self.rews_buf[self.ptr] = rew\n",
    "        self.done_buf[self.ptr] = done\n",
    "        self.ptr = (self.ptr + 1) % self.max_size\n",
    "        self.size = min(self.size + 1, self.max_size)\n",
    "        \n",
    "        return self.n_step_buffer[0]\n",
    "\n",
    "    def sample_batch(self) -> Dict[str, np.ndarray]:\n",
    "        idxs = np.random.choice(self.size, size=self.batch_size, replace=False)\n",
    "\n",
    "        return dict(\n",
    "            obs=self.obs_buf[idxs],\n",
    "            next_obs=self.next_obs_buf[idxs],\n",
    "            acts=self.acts_buf[idxs],\n",
    "            rews=self.rews_buf[idxs],\n",
    "            done=self.done_buf[idxs],\n",
    "            # for N-step Learning\n",
    "            indices=idxs,\n",
    "        )\n",
    "    \n",
    "    def sample_batch_from_idxs(\n",
    "        self, idxs: np.ndarray\n",
    "    ) -> Dict[str, np.ndarray]:\n",
    "        # for N-step Learning\n",
    "        return dict(\n",
    "            obs=self.obs_buf[idxs],\n",
    "            next_obs=self.next_obs_buf[idxs],\n",
    "            acts=self.acts_buf[idxs],\n",
    "            rews=self.rews_buf[idxs],\n",
    "            done=self.done_buf[idxs],\n",
    "        )\n",
    "    \n",
    "    def _get_n_step_info(\n",
    "        self, n_step_buffer: Deque, gamma: float\n",
    "    ) -> Tuple[np.int64, np.ndarray, bool]:\n",
    "        \"\"\"Return n step rew, next_obs, and done.\"\"\"\n",
    "        # info of the last transition\n",
    "        rew, next_obs, done = n_step_buffer[-1][-3:]\n",
    "\n",
    "        for transition in reversed(list(n_step_buffer)[:-1]):\n",
    "            r, n_o, d = transition[-3:]\n",
    "\n",
    "            rew = r + gamma * rew * (1 - d)\n",
    "            next_obs, done = (n_o, d) if d else (next_obs, done)\n",
    "\n",
    "        return rew, next_obs, done\n",
    "\n",
    "    def __len__(self) -> int:\n",
    "        return self.size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prioritized replay Buffer\n",
    "\n",
    "`store` method returns boolean in order to inform if a N-step transition has been generated.\n",
    "\n",
    "(Please see *02.per.ipynb* for detailed description about PER.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrioritizedReplayBuffer(ReplayBuffer):\n",
    "    \"\"\"Prioritized Replay buffer.\n",
    "    \n",
    "    Attributes:\n",
    "        max_priority (float): max priority\n",
    "        tree_ptr (int): next index of tree\n",
    "        alpha (float): alpha parameter for prioritized replay buffer\n",
    "        sum_tree (SumSegmentTree): sum tree for prior\n",
    "        min_tree (MinSegmentTree): min tree for min prior to get max weight\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(\n",
    "        self, \n",
    "        obs_dim: int, \n",
    "        size: int, \n",
    "        batch_size: int = 32, \n",
    "        alpha: float = 0.6,\n",
    "        n_step: int = 1, \n",
    "        gamma: float = 0.99,\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        assert alpha >= 0\n",
    "        \n",
    "        super(PrioritizedReplayBuffer, self).__init__(\n",
    "            obs_dim, size, batch_size, n_step, gamma\n",
    "        )\n",
    "        self.max_priority, self.tree_ptr = 1.0, 0\n",
    "        self.alpha = alpha\n",
    "        \n",
    "        # capacity must be positive and a power of 2.\n",
    "        tree_capacity = 1\n",
    "        while tree_capacity < self.max_size:\n",
    "            tree_capacity *= 2\n",
    "\n",
    "        self.sum_tree = SumSegmentTree(tree_capacity)\n",
    "        self.min_tree = MinSegmentTree(tree_capacity)\n",
    "        \n",
    "    def store(\n",
    "        self, \n",
    "        obs: np.ndarray, \n",
    "        act: int, \n",
    "        rew: float, \n",
    "        next_obs: np.ndarray, \n",
    "        done: bool,\n",
    "    ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n",
    "        \"\"\"Store experience and priority.\"\"\"\n",
    "        transition = super().store(obs, act, rew, next_obs, done)\n",
    "        \n",
    "        if transition:\n",
    "            self.sum_tree[self.tree_ptr] = self.max_priority ** self.alpha\n",
    "            self.min_tree[self.tree_ptr] = self.max_priority ** self.alpha\n",
    "            self.tree_ptr = (self.tree_ptr + 1) % self.max_size\n",
    "        \n",
    "        return transition\n",
    "\n",
    "    def sample_batch(self, beta: float = 0.4) -> Dict[str, np.ndarray]:\n",
    "        \"\"\"Sample a batch of experiences.\"\"\"\n",
    "        assert len(self) >= self.batch_size\n",
    "        assert beta > 0\n",
    "        \n",
    "        indices = self._sample_proportional()\n",
    "        \n",
    "        obs = self.obs_buf[indices]\n",
    "        next_obs = self.next_obs_buf[indices]\n",
    "        acts = self.acts_buf[indices]\n",
    "        rews = self.rews_buf[indices]\n",
    "        done = self.done_buf[indices]\n",
    "        weights = np.array([self._calculate_weight(i, beta) for i in indices])\n",
    "        \n",
    "        return dict(\n",
    "            obs=obs,\n",
    "            next_obs=next_obs,\n",
    "            acts=acts,\n",
    "            rews=rews,\n",
    "            done=done,\n",
    "            weights=weights,\n",
    "            indices=indices,\n",
    "        )\n",
    "        \n",
    "    def update_priorities(self, indices: List[int], priorities: np.ndarray):\n",
    "        \"\"\"Update priorities of sampled transitions.\"\"\"\n",
    "        assert len(indices) == len(priorities)\n",
    "\n",
    "        for idx, priority in zip(indices, priorities):\n",
    "            assert priority > 0\n",
    "            assert 0 <= idx < len(self)\n",
    "\n",
    "            self.sum_tree[idx] = priority ** self.alpha\n",
    "            self.min_tree[idx] = priority ** self.alpha\n",
    "\n",
    "            self.max_priority = max(self.max_priority, priority)\n",
    "            \n",
    "    def _sample_proportional(self) -> List[int]:\n",
    "        \"\"\"Sample indices based on proportions.\"\"\"\n",
    "        indices = []\n",
    "        p_total = self.sum_tree.sum(0, len(self) - 1)\n",
    "        segment = p_total / self.batch_size\n",
    "        \n",
    "        for i in range(self.batch_size):\n",
    "            a = segment * i\n",
    "            b = segment * (i + 1)\n",
    "            upperbound = random.uniform(a, b)\n",
    "            idx = self.sum_tree.retrieve(upperbound)\n",
    "            indices.append(idx)\n",
    "            \n",
    "        return indices\n",
    "    \n",
    "    def _calculate_weight(self, idx: int, beta: float):\n",
    "        \"\"\"Calculate the weight of the experience at idx.\"\"\"\n",
    "        # get max weight\n",
    "        p_min = self.min_tree.min() / self.sum_tree.sum()\n",
    "        max_weight = (p_min * len(self)) ** (-beta)\n",
    "        \n",
    "        # calculate weights\n",
    "        p_sample = self.sum_tree[idx] / self.sum_tree.sum()\n",
    "        weight = (p_sample * len(self)) ** (-beta)\n",
    "        weight = weight / max_weight\n",
    "        \n",
    "        return weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Noisy Layer\n",
    "\n",
    "Please see *05.noisy_net.ipynb* for detailed description.\n",
    "\n",
    "**References:**\n",
    "\n",
    "- https://github.com/higgsfield/RL-Adventure/blob/master/5.noisy%20dqn.ipynb\n",
    "- https://github.com/Kaixhin/Rainbow/blob/master/model.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NoisyLinear(nn.Module):\n",
    "    \"\"\"Noisy linear module for NoisyNet.\n",
    "    \n",
    "    \n",
    "        \n",
    "    Attributes:\n",
    "        in_features (int): input size of linear module\n",
    "        out_features (int): output size of linear module\n",
    "        std_init (float): initial std value\n",
    "        weight_mu (nn.Parameter): mean value weight parameter\n",
    "        weight_sigma (nn.Parameter): std value weight parameter\n",
    "        bias_mu (nn.Parameter): mean value bias parameter\n",
    "        bias_sigma (nn.Parameter): std value bias parameter\n",
    "        \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        in_features: int, \n",
    "        out_features: int, \n",
    "        std_init: float = 0.5,\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        super(NoisyLinear, self).__init__()\n",
    "        \n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.std_init = std_init\n",
    "\n",
    "        self.weight_mu = nn.Parameter(torch.Tensor(out_features, in_features))\n",
    "        self.weight_sigma = nn.Parameter(\n",
    "            torch.Tensor(out_features, in_features)\n",
    "        )\n",
    "        self.register_buffer(\n",
    "            \"weight_epsilon\", torch.Tensor(out_features, in_features)\n",
    "        )\n",
    "\n",
    "        self.bias_mu = nn.Parameter(torch.Tensor(out_features))\n",
    "        self.bias_sigma = nn.Parameter(torch.Tensor(out_features))\n",
    "        self.register_buffer(\"bias_epsilon\", torch.Tensor(out_features))\n",
    "\n",
    "        self.reset_parameters()\n",
    "        self.reset_noise()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        \"\"\"Reset trainable network parameters (factorized gaussian noise).\"\"\"\n",
    "        mu_range = 1 / math.sqrt(self.in_features)\n",
    "        self.weight_mu.data.uniform_(-mu_range, mu_range)\n",
    "        self.weight_sigma.data.fill_(\n",
    "            self.std_init / math.sqrt(self.in_features)\n",
    "        )\n",
    "        self.bias_mu.data.uniform_(-mu_range, mu_range)\n",
    "        self.bias_sigma.data.fill_(\n",
    "            self.std_init / math.sqrt(self.out_features)\n",
    "        )\n",
    "\n",
    "    def reset_noise(self):\n",
    "        \"\"\"Make new noise.\"\"\"\n",
    "        epsilon_in = self.scale_noise(self.in_features)\n",
    "        epsilon_out = self.scale_noise(self.out_features)\n",
    "\n",
    "        # outer product\n",
    "        self.weight_epsilon.copy_(epsilon_out.ger(epsilon_in))\n",
    "        self.bias_epsilon.copy_(epsilon_out)\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\n",
    "        \n",
    "        We don't use separate statements on train / eval mode.\n",
    "        It doesn't show remarkable difference of performance.\n",
    "        \"\"\"\n",
    "        return F.linear(\n",
    "            x,\n",
    "            self.weight_mu + self.weight_sigma * self.weight_epsilon,\n",
    "            self.bias_mu + self.bias_sigma * self.bias_epsilon,\n",
    "        )\n",
    "    \n",
    "    @staticmethod\n",
    "    def scale_noise(size: int) -> torch.Tensor:\n",
    "        \"\"\"Set scale to make noise (factorized gaussian noise).\"\"\"\n",
    "        x = torch.FloatTensor(np.random.normal(loc=0.0, scale=1.0, size=size))\n",
    "\n",
    "        return x.sign().mul(x.abs().sqrt())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NoisyNet + DuelingNet + Categorical DQN\n",
    "\n",
    "#### NoisyNet + DuelingNet\n",
    "\n",
    "NoisyLinear is employed for the last two layers of advantage and value layers. The noise should be reset at evey update step.\n",
    "\n",
    "#### DuelingNet + Categorical DQN\n",
    "\n",
    "The dueling network architecture is adapted for use with return distributions. The network has a shared representation, which is then fed into a value stream with atom_size outputs, and into an advantage stream with atom_size × out_dim outputs. For each atom, the value and advantage streams are aggregated, as in dueling DQN, and then passed through a softmax layer to obtain the normalized parametric distributions used to estimate the returns’ distributions.\n",
    "\n",
    "```\n",
    "        advantage = self.advantage_layer(adv_hid).view(-1, self.out_dim, self.atom_size)\n",
    "        value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n",
    "        q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n",
    "        \n",
    "        dist = F.softmax(q_atoms, dim=-1)\n",
    "```\n",
    "\n",
    "(Please see *04.dueling.ipynb*, *05.noisy_net.ipynb*, *06.categorical_dqn.ipynb* for detailed description of each component's network architecture.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        in_dim: int, \n",
    "        out_dim: int, \n",
    "        atom_size: int, \n",
    "        support: torch.Tensor\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        super(Network, self).__init__()\n",
    "        \n",
    "        self.support = support\n",
    "        self.out_dim = out_dim\n",
    "        self.atom_size = atom_size\n",
    "\n",
    "        # set common feature layer\n",
    "        self.feature_layer = nn.Sequential(\n",
    "            nn.Linear(in_dim, 128), \n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        # set advantage layer\n",
    "        self.advantage_hidden_layer = NoisyLinear(128, 128)\n",
    "        self.advantage_layer = NoisyLinear(128, out_dim * atom_size)\n",
    "\n",
    "        # set value layer\n",
    "        self.value_hidden_layer = NoisyLinear(128, 128)\n",
    "        self.value_layer = NoisyLinear(128, atom_size)\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\"\"\"\n",
    "        dist = self.dist(x)\n",
    "        q = torch.sum(dist * self.support, dim=2)\n",
    "        \n",
    "        return q\n",
    "    \n",
    "    def dist(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Get distribution for atoms.\"\"\"\n",
    "        feature = self.feature_layer(x)\n",
    "        adv_hid = F.relu(self.advantage_hidden_layer(feature))\n",
    "        val_hid = F.relu(self.value_hidden_layer(feature))\n",
    "        \n",
    "        advantage = self.advantage_layer(adv_hid).view(\n",
    "            -1, self.out_dim, self.atom_size\n",
    "        )\n",
    "        value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n",
    "        q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n",
    "        \n",
    "        dist = F.softmax(q_atoms, dim=-1)\n",
    "        dist = dist.clamp(min=1e-3)  # for avoiding nans\n",
    "        \n",
    "        return dist\n",
    "    \n",
    "    def reset_noise(self):\n",
    "        \"\"\"Reset all noisy layers.\"\"\"\n",
    "        self.advantage_hidden_layer.reset_noise()\n",
    "        self.advantage_layer.reset_noise()\n",
    "        self.value_hidden_layer.reset_noise()\n",
    "        self.value_layer.reset_noise()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rainbow Agent\n",
    "\n",
    "Here is a summary of DQNAgent class.\n",
    "\n",
    "| Method           | Note                                                 |\n",
    "| ---              | ---                                                  |\n",
    "|select_action     | select an action from the input state.               |\n",
    "|step              | take an action and return the response of the env.   |\n",
    "|compute_dqn_loss  | return dqn loss.                                     |\n",
    "|update_model      | update the model by gradient descent.                |\n",
    "|target_hard_update| hard update from the local model to the target model.|\n",
    "|train             | train the agent during num_frames.                   |\n",
    "|test              | test the agent (1 episode).                          |\n",
    "|plot              | plot the training progresses.                        |\n",
    "\n",
    "#### Categorical DQN + Double DQN\n",
    "\n",
    "The idea of Double Q-learning is to reduce overestimations by decomposing the max operation in the target into action selection and action evaluation. Here, we use `self.dqn` instead of `self.dqn_target` to obtain the target actions.\n",
    "\n",
    "```\n",
    "        # Categorical DQN + Double DQN\n",
    "        # target_dqn is used when we don't employ double DQN\n",
    "        next_action = self.dqn(next_state).argmax(1)\n",
    "        next_dist = self.dqn_target.dist(next_state)\n",
    "        next_dist = next_dist[range(self.batch_size), next_action]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    \"\"\"DQN Agent interacting with environment.\n",
    "    \n",
    "    Attribute:\n",
    "        env (gym.Env): openAI Gym environment\n",
    "        memory (PrioritizedReplayBuffer): replay memory to store transitions\n",
    "        batch_size (int): batch size for sampling\n",
    "        target_update (int): period for target model's hard update\n",
    "        gamma (float): discount factor\n",
    "        dqn (Network): model to train and select actions\n",
    "        dqn_target (Network): target model to update\n",
    "        optimizer (torch.optim): optimizer for training dqn\n",
    "        transition (list): transition information including \n",
    "                           state, action, reward, next_state, done\n",
    "        v_min (float): min value of support\n",
    "        v_max (float): max value of support\n",
    "        atom_size (int): the unit number of support\n",
    "        support (torch.Tensor): support for categorical dqn\n",
    "        use_n_step (bool): whether to use n_step memory\n",
    "        n_step (int): step number to calculate n-step td error\n",
    "        memory_n (ReplayBuffer): n-step replay buffer\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        env: gym.Env,\n",
    "        memory_size: int,\n",
    "        batch_size: int,\n",
    "        target_update: int,\n",
    "        gamma: float = 0.99,\n",
    "        # PER parameters\n",
    "        alpha: float = 0.2,\n",
    "        beta: float = 0.6,\n",
    "        prior_eps: float = 1e-6,\n",
    "        # Categorical DQN parameters\n",
    "        v_min: float = 0.0,\n",
    "        v_max: float = 200.0,\n",
    "        atom_size: int = 51,\n",
    "        # N-step Learning\n",
    "        n_step: int = 3,\n",
    "    ):\n",
    "        \"\"\"Initialization.\n",
    "        \n",
    "        Args:\n",
    "            env (gym.Env): openAI Gym environment\n",
    "            memory_size (int): length of memory\n",
    "            batch_size (int): batch size for sampling\n",
    "            target_update (int): period for target model's hard update\n",
    "            lr (float): learning rate\n",
    "            gamma (float): discount factor\n",
    "            alpha (float): determines how much prioritization is used\n",
    "            beta (float): determines how much importance sampling is used\n",
    "            prior_eps (float): guarantees every transition can be sampled\n",
    "            v_min (float): min value of support\n",
    "            v_max (float): max value of support\n",
    "            atom_size (int): the unit number of support\n",
    "            n_step (int): step number to calculate n-step td error\n",
    "        \"\"\"\n",
    "        obs_dim = env.observation_space.shape[0]\n",
    "        action_dim = env.action_space.n\n",
    "        \n",
    "        self.env = env\n",
    "        self.batch_size = batch_size\n",
    "        self.target_update = target_update\n",
    "        self.gamma = gamma\n",
    "        # NoisyNet: All attributes related to epsilon are removed\n",
    "        \n",
    "        # device: cpu / gpu\n",
    "        self.device = torch.device(\n",
    "            \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "        )\n",
    "        print(self.device)\n",
    "        \n",
    "        # PER\n",
    "        # memory for 1-step Learning\n",
    "        self.beta = beta\n",
    "        self.prior_eps = prior_eps\n",
    "        self.memory = PrioritizedReplayBuffer(\n",
    "            obs_dim, memory_size, batch_size, alpha=alpha\n",
    "        )\n",
    "        \n",
    "        # memory for N-step Learning\n",
    "        self.use_n_step = True if n_step > 1 else False\n",
    "        if self.use_n_step:\n",
    "            self.n_step = n_step\n",
    "            self.memory_n = ReplayBuffer(\n",
    "                obs_dim, memory_size, batch_size, n_step=n_step, gamma=gamma\n",
    "            )\n",
    "            \n",
    "        # Categorical DQN parameters\n",
    "        self.v_min = v_min\n",
    "        self.v_max = v_max\n",
    "        self.atom_size = atom_size\n",
    "        self.support = torch.linspace(\n",
    "            self.v_min, self.v_max, self.atom_size\n",
    "        ).to(self.device)\n",
    "\n",
    "        # networks: dqn, dqn_target\n",
    "        self.dqn = Network(\n",
    "            obs_dim, action_dim, self.atom_size, self.support\n",
    "        ).to(self.device)\n",
    "        self.dqn_target = Network(\n",
    "            obs_dim, action_dim, self.atom_size, self.support\n",
    "        ).to(self.device)\n",
    "        self.dqn_target.load_state_dict(self.dqn.state_dict())\n",
    "        self.dqn_target.eval()\n",
    "        \n",
    "        # optimizer\n",
    "        self.optimizer = optim.Adam(self.dqn.parameters())\n",
    "\n",
    "        # transition to store in memory\n",
    "        self.transition = list()\n",
    "        \n",
    "        # mode: train / test\n",
    "        self.is_test = False\n",
    "\n",
    "    def select_action(self, state: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"Select an action from the input state.\"\"\"\n",
    "        # NoisyNet: no epsilon greedy action selection\n",
    "        selected_action = self.dqn(\n",
    "            torch.FloatTensor(state).to(self.device)\n",
    "        ).argmax()\n",
    "        selected_action = selected_action.detach().cpu().numpy()\n",
    "        \n",
    "        if not self.is_test:\n",
    "            self.transition = [state, selected_action]\n",
    "        \n",
    "        return selected_action\n",
    "\n",
    "    def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:\n",
    "        \"\"\"Take an action and return the response of the env.\"\"\"\n",
    "        next_state, reward, done, _ = self.env.step(action)\n",
    "\n",
    "        if not self.is_test:\n",
    "            self.transition += [reward, next_state, done]\n",
    "            \n",
    "            # N-step transition\n",
    "            if self.use_n_step:\n",
    "                one_step_transition = self.memory_n.store(*self.transition)\n",
    "            # 1-step transition\n",
    "            else:\n",
    "                one_step_transition = self.transition\n",
    "\n",
    "            # add a single step transition\n",
    "            if one_step_transition:\n",
    "                self.memory.store(*one_step_transition)\n",
    "    \n",
    "        return next_state, reward, done\n",
    "\n",
    "    def update_model(self) -> torch.Tensor:\n",
    "        \"\"\"Update the model by gradient descent.\"\"\"\n",
    "        # PER needs beta to calculate weights\n",
    "        samples = self.memory.sample_batch(self.beta)\n",
    "        weights = torch.FloatTensor(\n",
    "            samples[\"weights\"].reshape(-1, 1)\n",
    "        ).to(self.device)\n",
    "        indices = samples[\"indices\"]\n",
    "        \n",
    "        # 1-step Learning loss\n",
    "        elementwise_loss = self._compute_dqn_loss(samples, self.gamma)\n",
    "        \n",
    "        # PER: importance sampling before average\n",
    "        loss = torch.mean(elementwise_loss * weights)\n",
    "        \n",
    "        # N-step Learning loss\n",
    "        # we are gonna combine 1-step loss and n-step loss so as to\n",
    "        # prevent high-variance. The original rainbow employs n-step loss only.\n",
    "        if self.use_n_step:\n",
    "            gamma = self.gamma ** self.n_step\n",
    "            samples = self.memory_n.sample_batch_from_idxs(indices)\n",
    "            elementwise_loss_n_loss = self._compute_dqn_loss(samples, gamma)\n",
    "            elementwise_loss += elementwise_loss_n_loss\n",
    "            \n",
    "            # PER: importance sampling before average\n",
    "            loss = torch.mean(elementwise_loss * weights)\n",
    "\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        clip_grad_norm_(self.dqn.parameters(), 10.0)\n",
    "        self.optimizer.step()\n",
    "        \n",
    "        # PER: update priorities\n",
    "        loss_for_prior = elementwise_loss.detach().cpu().numpy()\n",
    "        new_priorities = loss_for_prior + self.prior_eps\n",
    "        self.memory.update_priorities(indices, new_priorities)\n",
    "        \n",
    "        # NoisyNet: reset noise\n",
    "        self.dqn.reset_noise()\n",
    "        self.dqn_target.reset_noise()\n",
    "\n",
    "        return loss.item()\n",
    "        \n",
    "    def train(self, num_frames: int, plotting_interval: int = 200):\n",
    "        \"\"\"Train the agent.\"\"\"\n",
    "        self.is_test = False\n",
    "        \n",
    "        state = self.env.reset()\n",
    "        update_cnt = 0\n",
    "        losses = []\n",
    "        scores = []\n",
    "        score = 0\n",
    "\n",
    "        for frame_idx in range(1, num_frames + 1):\n",
    "            action = self.select_action(state)\n",
    "            next_state, reward, done = self.step(action)\n",
    "\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            \n",
    "            # NoisyNet: removed decrease of epsilon\n",
    "            \n",
    "            # PER: increase beta\n",
    "            fraction = min(frame_idx / num_frames, 1.0)\n",
    "            self.beta = self.beta + fraction * (1.0 - self.beta)\n",
    "\n",
    "            # if episode ends\n",
    "            if done:\n",
    "                state = self.env.reset()\n",
    "                scores.append(score)\n",
    "                score = 0\n",
    "\n",
    "            # if training is ready\n",
    "            if len(self.memory) >= self.batch_size:\n",
    "                loss = self.update_model()\n",
    "                losses.append(loss)\n",
    "                update_cnt += 1\n",
    "                \n",
    "                # if hard update is needed\n",
    "                if update_cnt % self.target_update == 0:\n",
    "                    self._target_hard_update()\n",
    "\n",
    "            # plotting\n",
    "            if frame_idx % plotting_interval == 0:\n",
    "                self._plot(frame_idx, scores, losses)\n",
    "                \n",
    "        self.env.close()\n",
    "                \n",
    "    def test(self) -> List[np.ndarray]:\n",
    "        \"\"\"Test the agent.\"\"\"\n",
    "        self.is_test = True\n",
    "        \n",
    "        state = self.env.reset()\n",
    "        done = False\n",
    "        score = 0\n",
    "        \n",
    "        frames = []\n",
    "        while not done:\n",
    "            frames.append(self.env.render(mode=\"rgb_array\"))\n",
    "            action = self.select_action(state)\n",
    "            next_state, reward, done = self.step(action)\n",
    "\n",
    "            state = next_state\n",
    "            score += reward\n",
    "        \n",
    "        print(\"score: \", score)\n",
    "        self.env.close()\n",
    "        \n",
    "        return frames\n",
    "\n",
    "    def _compute_dqn_loss(self, samples: Dict[str, np.ndarray], gamma: float) -> torch.Tensor:\n",
    "        \"\"\"Return categorical dqn loss.\"\"\"\n",
    "        device = self.device  # for shortening the following lines\n",
    "        state = torch.FloatTensor(samples[\"obs\"]).to(device)\n",
    "        next_state = torch.FloatTensor(samples[\"next_obs\"]).to(device)\n",
    "        action = torch.LongTensor(samples[\"acts\"]).to(device)\n",
    "        reward = torch.FloatTensor(samples[\"rews\"].reshape(-1, 1)).to(device)\n",
    "        done = torch.FloatTensor(samples[\"done\"].reshape(-1, 1)).to(device)\n",
    "        \n",
    "        # Categorical DQN algorithm\n",
    "        delta_z = float(self.v_max - self.v_min) / (self.atom_size - 1)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            # Double DQN\n",
    "            next_action = self.dqn(next_state).argmax(1)\n",
    "            next_dist = self.dqn_target.dist(next_state)\n",
    "            next_dist = next_dist[range(self.batch_size), next_action]\n",
    "\n",
    "            t_z = reward + (1 - done) * gamma * self.support\n",
    "            t_z = t_z.clamp(min=self.v_min, max=self.v_max)\n",
    "            b = (t_z - self.v_min) / delta_z\n",
    "            l = b.floor().long()\n",
    "            u = b.ceil().long()\n",
    "\n",
    "            offset = (\n",
    "                torch.linspace(\n",
    "                    0, (self.batch_size - 1) * self.atom_size, self.batch_size\n",
    "                ).long()\n",
    "                .unsqueeze(1)\n",
    "                .expand(self.batch_size, self.atom_size)\n",
    "                .to(self.device)\n",
    "            )\n",
    "\n",
    "            proj_dist = torch.zeros(next_dist.size(), device=self.device)\n",
    "            proj_dist.view(-1).index_add_(\n",
    "                0, (l + offset).view(-1), (next_dist * (u.float() - b)).view(-1)\n",
    "            )\n",
    "            proj_dist.view(-1).index_add_(\n",
    "                0, (u + offset).view(-1), (next_dist * (b - l.float())).view(-1)\n",
    "            )\n",
    "\n",
    "        dist = self.dqn.dist(state)\n",
    "        log_p = torch.log(dist[range(self.batch_size), action])\n",
    "        elementwise_loss = -(proj_dist * log_p).sum(1)\n",
    "\n",
    "        return elementwise_loss\n",
    "\n",
    "    def _target_hard_update(self):\n",
    "        \"\"\"Hard update: target <- local.\"\"\"\n",
    "        self.dqn_target.load_state_dict(self.dqn.state_dict())\n",
    "                \n",
    "    def _plot(\n",
    "        self, \n",
    "        frame_idx: int, \n",
    "        scores: List[float], \n",
    "        losses: List[float],\n",
    "    ):\n",
    "        \"\"\"Plot the training progresses.\"\"\"\n",
    "        clear_output(True)\n",
    "        plt.figure(figsize=(20, 5))\n",
    "        plt.subplot(131)\n",
    "        plt.title('frame %s. score: %s' % (frame_idx, np.mean(scores[-10:])))\n",
    "        plt.plot(scores)\n",
    "        plt.subplot(132)\n",
    "        plt.title('loss')\n",
    "        plt.plot(losses)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Environment\n",
    "\n",
    "You can see the [code](https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py) and [configurations](https://github.com/openai/gym/blob/master/gym/envs/__init__.py#L53) of CartPole-v0 from OpenAI's repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# environment\n",
    "env_id = \"CartPole-v0\"\n",
    "env = gym.make(env_id)\n",
    "if IN_COLAB:\n",
    "    env = gym.wrappers.Monitor(env, \"videos\", force=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[777]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed = 777\n",
    "\n",
    "def seed_torch(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.backends.cudnn.enabled:\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "seed_torch(seed)\n",
    "env.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "# parameters\n",
    "num_frames = 20000\n",
    "memory_size = 1000\n",
    "batch_size = 32\n",
    "target_update = 100\n",
    "\n",
    "# train\n",
    "agent = DQNAgent(env, memory_size, batch_size, target_update)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv4AAAE/CAYAAAA+Occ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXmcHGW1/p/T62zZMwkBskICgoEAYVF22UVBvVcRvCouF7nqVUC9ovhzvXpxRbkqCKK4IgIieEH2fQmQQBLCmoWE7DOZSWbvrfr9/VFLV/dU71VdvTzfz2c+6a56u+qd7knXeU895zmilAIhhBBCCCGkuQn4PQFCCCGEEEKI9zDwJ4QQQgghpAVg4E8IIYQQQkgLwMCfEEIIIYSQFoCBPyGEEEIIIS0AA39CCCGEEEJaAAb+TYyIHCAiK0VkSEQ+5/d8CCGEkHpGRDaKyKl+z4MQr2Dg39z8F4CHlVITlFJX+z0ZOyKySETuEJFeEekXkXtF5ICcMZeKyA4RGRSR34hI1LZvnog8LCKjIvJq7hd1Na9tZETkrcZ7uUtExjXpEJG3iMhDIjIgIutE5L15jvN1EVGF3hsRebuIPGssLFeLyHFu/i6EEEIIcRcG/s3NXAAv5dspIsEaziWXyQDuBHAAgJkAngVwh7lTRM4AcDmAU6D/HgsAfMv2+psAvABgGoArANwqIt3VvtZPRKfa/5NJAH8F8AmH44egv8f/B2AqgIsA/FFEFuWM2w/A+wFsLzDXqQD+AeCH0D/LHwD4h4hMqXL+hBBCCPEIBv5Niog8BOBkAD8XkWEjw36jiFwjIneLyAiAk0XkbBF5wciMbxaRb9qOMc/I+n7M2LdbRC4WkSONDO8eEfl5znk/LiKvGGPvFZG5TvNTSj2rlLpBKdWvlEoCuArAASIyzRjyUQA3KKVeUkrtBvAdABca51gE4HAA31BKjSmlbgPwIoB/ceG1xd7XL4vIViPL/ZqInGJsD4rIV0VkvbFvhYjMNva9XUSeM7Lsz4nI223He0REvisiTwIYBbBARCaJyA0ist0413+XukhTSr2mlLoBzgu+AwHsDeAqpZSmlHoIwJMAPpwz7hcAvgwgUeBUbwewQyl1i3GsPwLoBfC+UuZJCCH1jIhEReSnIrLN+PmpeedYRKaLyP8Z18B+EXncTNrku0YQUi8w8G9SlFLvAPA4gM8qpbqUUq8buy4A8F0AEwA8AWAEwEegZ23PBvAfIvKenMMdDWAhgPMA/BR6lvxUAAcD+ICInAgAInIugK9CD/66jfPfVOKUT4AeSPYZzw8GsMq2fxWAmcbC4GAAG5RSQzn7D3bhtXkRXYr0WQBHKqUmADgDwEZj92UAzgfwTgATAXwcwKiRGb8LwNXQ7zD8BMBdtgUOoAfeF0H/TDYBuBFACsD+AA4DcDqATxpzmGNcbOYUm2+JCIC32n7H9wOIK6XuLvG1eY9FCCENzBUAjgGwBMChAI4C8DVj3xcAbIF+nZsJ/bqnilwjCKkLGPi3HncopZ5USqWVUjGl1CNKqReN56uhB+on5rzmO8bY+6AvFG5SSvUopbZCD+4PM8ZdDOB/lFKvKKVSAL4HYEm+rL+JiOwLPct8mW1zF4AB23Pz8QSHfeb+CS68thAagCiAg0QkrJTaqJRab+z7JICvGRl3pZRaZSxizgawVin1B6VUSil1E4BXAbzbdtwbjbsTKegSnHcCuEQpNaKU6oF+N+SDAKCUelMpNVkp9WYJ883lNQA9AL4kImEROR36Z90BACIyAfpn9vkSjvU0gL1F5HzjWB8FsJ95LEIIaXA+BODbxrWuF7pc1Lw7mgQwC8BcpVRSKfW4Ukqh8DWCkLqAgX/rsdn+RESOFr3QtVdEBqAH79NzXrPT9njM4XmX8XgugJ8ZGek9APqhZ4H3yTcZQ1t/H4BfGkGxyTD0zLmJ+XjIYZ+538ziV/PavCil1gG4BMA3AfSIyF9EZG9j92wATl/we0PP4tvZhOz3xP6ZzAUQBrDd9j7+CsCMYvMrYf5JAO+BvhjZAT1r9VfomStA/73+oJTaWMKx+gCcC32xthPAmQAesB2LEEIamdzv7k3GNkCvbVoH4D4R2SAilwNFrxGE1AUM/FuPXKeXP0Mvsp2tlJoE4FqMl3CUymYAnzIy0uZPu1LqKafBRiHofQDuVEp9N2f3S9Bvr5ocCmCnEXC+BF0LPyFn/0suvLYgSqk/K6WOgx6gKwDft/3u+zm8ZJsx1s4cAFvth7U93gwgDmC67T2cqJQqKkUqcf6rlVInKqWmKaXOgF74/Kyx+xQAnxPdDWkH9MXMX0Xky3mO9ahS6kil1FTombADbccihJBGJve7e46xDUqpIaXUF5RSCwCcA+AyU8tf4BpBSF3AwJ9MANCvlIqJyFHQawAq5VoAXxGRgwHAKFJ9v9NAEZkI4F4ATyqlLncY8nsAnxCRg0RkMnRt5Y0AYNQrrATwDRFpE92S8hAAt7nw2ryI3hfhHUaBVwz63Y60sfvXAL4jIgtF5xBDx383gEUicoGIhETkPAAHQXfWGYdSajv0xdCPRWSiiAREZD+zjqKEOYqItAGIGM/bJNvK9BBjW4eIfBH67eobjd2nQNfoLzF+tgH4FHQZltO5DjNkPhMB/AjAZqXUvaXMkxBC6pybAHxNRLpFZDqArwP4IwCIyLtEZH8REehSUQ1Ausg1gpC6gIE/+TSAb4vIEPQvtr9WeiCl1O3Qsxt/EZFBAGsAnJVn+HsBHAngY6K7Dpk/c4xj3QPdIvJhAG9Cv836DdvrPwhgKYDdAK4E8K+GDrOq14rIh0QkX/Y/aozfBV0qMwPAV4x9P4H+3t0HYBDADQDajbsM74Iuq+mD3lvhXUqpXXnOAejF1hEALxtzvBV6gG4W91rvkwNzoV9szN9hDLq23+TD0G06e6AH+qcppeKALt9RSu0wf6BfzHYrpYaNc18rItfajvVfxnux2ZifY08AQghpQP4bwHIAq6E7vz1vbAN0s4sHoEtHn4YuVX0Yha8RhNQFotejEEIIIYQQQpoZZvwJIYQQQghpARj4E0IIIYQQ0gIw8CeEEEIIIaQFYOBPCCGEEEJIC8DAnxBCCCGEkBYg5PcEAGD69Olq3rx5fk+DEELqkhUrVuxSSnX7PQ8/4XWCEEKcKecaUReB/7x587B8+XK/p0EIIXWJiGzyew5+w+sEIYQ4U841glIfQgghhBBCWgAG/oQQQgghhLQADPwJIYQQQghpARj4E0IIIYQQ0gIw8CeEEEIIIaQFYOBPCCGEEEJIC8DAnxBCCCGEkBagaOAvIrNF5GEReVlEXhKRzxvbp4rI/SKy1vh3irFdRORqEVknIqtF5HCvfwlCCCGEEEJIYUrJ+KcAfEEpdRCAYwB8RkQOAnA5gAeVUgsBPGg8B4CzACw0fi4CcI3rsyaEEEIIIYSURdHOvUqp7QC2G4+HROQVAPsAOBfAScaw3wF4BMCXje2/V0opAMtEZLKIzDKOQxzY0DuMUCCAOdM68o5JpNJ4cv0unLSoGyICANg1HMf2PTEs3neSNW7N1gG8uHUAALDP5HacsCjTwfnNvlGk0mks6O6ytj21fhc29Y26/StZTGwL452L97Lm7Cbb9ozh0dd7XT9uvjkrpXDXi9sxFEu5fs5WpNq/jXU9w3huY3/RcZ3REM5ePAvBgH6evuE4HnhlJ9Iqe9wxC6Zh/vRO6/kjr/Vg+0Csork5ceS8qdh/RlfxgcRVnl7fh56hGM5dso/fUyGEEN8pGvjbEZF5AA4D8AyAmbZgfgeAmcbjfQBstr1si7EtK/AXkYug3xHAnDlzypx2c3H5315ERySIGz92VN4xP3vwdfzi4fW463PH4eC99UD/V4+ux83Pbcbqb55hjbv05pVY2zMMAAgI8NK3zkR7JAgA+MadazCa0HDzp94GQA9kL/ztc0ik0l79agCAey45HgfuNdH141794Fr85bnNxQdWgP19Nnlp2yA+++cXPDlfq3LfpSdg0cwJFb3263eswVPr+0oau++Udhw+ZwoA4ManNuJ/H1o3bsypb5mJX390KQBgOJ7Cx258DkqNG1YxV75vMQN/H7h1xRYs29DHwJ8QQlBG4C8iXQBuA3CJUmrQnqVTSikRKesSqZS6DsB1ALB06VIXL6+Nx57RBOLJYN7963qGcd1jGwAAA6NJa3v/SBKDsRSG4yl0RfWPcvtADOctnY150zvx/XtexVAsaQX+/aNJpLRMkK+lFRKpND51wgJ87Nj5rv9eT63fhcv+ugqxpDcLi9GEhn2ntOPWi9/u2jFXbdmDT/1hBfqGE+P2DY7p7/01HzochxlBJKmMZRv6cMnNKzEcr/zuyY7BGE45cAa++97FeceYn+eI7TxDMf3/ywOXnWht+48/rcBQLPN/aySeglLAl888EO89zJ2AcWJ7WXkWQgghxHVKuhKJSBh60P8npdTfjM07TQmPiMwC0GNs3wpgtu3l+xrbSB5GExq0XN2BgVIK/+/va5Ay9tsDpdGE/rhnMIau7i6MJvRFwNzpHZg1qc0aP8McnxNkJYxFwJTOCPYyxrvJtK4oAOT93aolqaXRHg66OvfBmC71GBhLjts3ktAAAPtO6fDk/WolZk7U3794FYvCvuEEjtt/esHPomdo/HniqTTacv5uJrSFsz7zWFL/rGdMiPKzJoQQ0jSU4uojAG4A8IpS6ie2XXcC+Kjx+KMA7rBt/4jh7nMMgAHq+wszltAwEtcc9925ahue3tCHjxsZ+dFEZpy5COgdimf9290VRYeR5bePH4mnkLRl/JMpPSCPBL1xdQ0ad4W8DPzDLs99cnsYgHPgby60zDsopHKiYf1zi6ec/+6LkUilMTCWxHRjcZn3PCH9s0rY/u4TqTSioey/m7ZQAPFkZi7mXaq2MD9rQgghzUMpUdOxAD4M4B0istL4eSeAKwGcJiJrAZxqPAeAuwFsALAOwPUAPu3+tJuLsaSWV/Jw58ptmDutA588Xg/87eNM+UKPEfCb/86Y2GZJf+zjh+MpJLVMEG4GQ+GQR4F/wOvAX7k+94kFAn9zcdYZZTBYLeZiM15hfUn/iC7FmtYVKTjODPDtC4x4SrMWHiZt4aCV5QcyGf+2MFudEEIIaR5KcfV5AkA+241THMYrAJ+pcl4tg1IKY0aQkU4rBALZb/XOoRgWTO/EhDY9IB3Jkvror8vN+M+YELUKds3xSimMJjREQpmg1Qz8o15l/I3fJe1mhaSNpJZGOOCuW1BbOIhoKFAw498RoVa7WtrC1QX+u4b1v/VpnUUy/uZ5cqQ+0VD24q09HMyqRTED/3Zm/AkhhDQRTGf5TCyZhlKAUsBocrzsYedgHDMntqHDCEBGHKQ+VsZ/ULce7J4QtbLS5vh4Ko1UWuVIfcyMv/tWm0Am8E81kNQHACZ3hLOKqE3MhVYHpT5VYwbecYe/+VLoMzL+3ROKZfyN86RyA//cjH/AWoADsB5HGfgTQghpIhj4+4yZRQays/kAkNLS2DUcx4yJbQgEBB2RYNaYjNQnZvwbRyggmNoRQach9THHmP/arTstqY/XGX+PAv+EB1IfAJjUHsaesfGuPiOJFCLBgGfvVyuRkeBUmPEfKi3jH3GS+iQ1a7vJeKmPqfHnZ00IIaR54FXNZ5yKdU16h+NQCpg5UQ9uOqOh7MDfQeozvSuKQEDGBf7meZI5RY5A4xb3prQ0IkH371ZMag87Sn3GEho6qO93BavotsLAv2/ECPxL1PjnLnhzM/7RcBDxVBrKkKWZCwUW9xJCCGkmGPj7jF1eMJzTEXbnoB7czJyg2wl2RUNWsJ/U0lYw02sr7u2eoC8SLGmQUZBqLipSaWVl4GtV3Oul1CcU8CLjH8HA2Phi65G4hk7q+10hWqXGv284gUgoYBWx5yMUEAQkR+qTHK/xz605yBT3MvAnhBDSPDDw95lcu007Ow3Nvul5bpf6mP+KZLv6zDAC/1AwgLZwACOJ1LhjmwG/qfFv3OJe76Q+A6PjpT6jiRStPF0i4+pTmca/dziO7q4o7I0EnRARREPBHI2/g6uPsRAYM/4/WlIfjxbFhBBCiB/wquYzdo1/rtSnxwr8M1Ifc4yZ+d97Ujv6RxJIamn0DsUxY2JG89zlMB7IyH2aIeMfrqHUZyShoZOBvysEAoJIMFBVxr+YzMckGs726Hcq7jUXdLGUGfhrWdsJIYSQZoCBv8+MFdD47xyMIxgQqwNuVzRkLRTMDP686R3G2Bj6RvQsqElHJGR1683K+BvBVrLBi3t1O09vXH1GElpWPQQAjCVStPJ0kWgoUHHn3r6ROKZ1lhb45y4w8rn6AJlMvynBawsx8G90itwUIoSQloKBv8/YNf5OUp/urqgVQOvFvdma/fnTOwEAr2wfglJAtyELMscP54wHYDXxSjR8517liRXppDxNvEbiGpt3uUg0HKhY6rNrKFG0a2/2eXI79+Zo/I3nZqY/lkwjEgyM66tBCCGENDIM/H3GrvEfyg38h+KWzAcAOiNBK4AfNQL6edP0wP+lbQMAYGn8AaArmqkJGHXI+JtSn4hXPv5BjwP/lDc+/vkCf13jz4y/W+Rq70tFKaVn/EsN/EPBLFefeEpzyPjnBv7j6wAIIYSQRodXNp8ZK1Dc2zMYw4ycDL4ZwOdm/F/aNggAlqsPYEh9EuM1/rnFvZGgN1lsK+PvVXFvOu3J3YpJHXrgv2c0N/Cnxt9NoqHKNP6DsRSSmsL0UjX+ocydBaWUo9THDPLNO3DxlEZHH0IIIU0HA3+fMTP+kWDAkvGY7ByMZWf8DTvPdFpZi4S5ZsZ/q1PGP1Pcmy31yS3u9Sbjb8rvvSvuVQh5VNwLAIPjMv4aNf4uEgkFKurcu2tYd7EqWepjW2AkNQWlxnfkbQ+bnYRNO8+0tY0QQghpFhj4+8yYkZGf1hXBkM3HP57SsHs0aXn4A7CyzaNJzcrkT+4IY2pnBNsGdAcge8a/Mxq0FhNOUh9zAeCVxt/02PeiuDedVtDSqmZSH6UURhIpdDDj7xpm06xy6RvWrVZLdvUJBa2A3sz8F5P6jCU0du0lhBDSdPDK5jOjCQ3t4SAmtGV35e0xm3flSH0APYg3i3Y7IyHLyWdyRziraLEjErJ8/IfjDnaeKY/tPD0s7k2mvXMkmtxuSn0yXv6xZBpKgZ17XcQuwSmHPiPjP62ztIx/xHYeq1t1vsDftPOk1IcQQkgTwsDfZ0aTGjoiQaMrry3wH9Iz+Lm+/IAu2xmJpxAQ3YbQHNOdI33oiuqLCaWUo51nwuOMv5fFvaYzkRc+/hOtjH/mPTPvsLBzr3uUovG/9OaV+OUj67K2ZaQ+5Wj8zYx/2tpmJ9fOM5bUaOVJCCGk6WAU4zNjCQ3tkSA6o6Esqc9Oh4y/KTMZiWsYSaTQGQ1BRCx5j32RAOh3CNJKD2ZGEimIAErZi3vN4NnjjL8Hxb1mYbIXcw8HA+iMBLOkPmYtBqU+7hENBS3ZTj6e29g/ruh9l/GaqSX6+EfDGVefTODvbOdp79w7oY1fj4QQQpoLZvx9ZtTQjZvZeZOdVtfeTOBvZvxHEnrG38w+zzDqAGbY6gEAWJ7z5nhTu275+GsaggGx+gS4jVnc603G39vmY5M7ItgzlglKzbsxLO51j1wf/96hONYYReomsWQasZy7An0jcUzpCCNU4mefnfF31vg7de5lcS8hhJBmg4G/z4wmNLRHQlkOPICe8Q8HBVMMa0kgo/Efiacwksg0kzIz/vbCXiAjSxmJpzCa0DClQ8+QJmwOJ17JfIBMca83Gn9vm49NbA9nufqYRdLU+LtHrtTnl4+sw8dvfC5rTDylWQW3JruGEiV7+GfOY9h0GlKeXI9+cyEQS2buDFDjTwghpNlg4O8zsaSGjnDQ6LJrL+6NYcaENoit37wZ6Jsaf/MOgGnhOSM38LfVBAzHU5jcYWb8M8W9XmjkTcwbCZ4E/kbA6IWdJwBMag9l+fibEhBq/N0jt4HXwGhyXNO0eDI9zvKzbyResr4fMG1Dc+tasoN6EdEXCHT1aUqUR71ECCGk0eCVzWd0b/iMq495gdo5lO3hD9gz/hpG4ilLdjIjX8Y/Gswan5vxT2hpRDwsYBTRZUQNKfVpj2QFoRmpD7PAbhHN8fEfTWiIp9LW/wEtrZDQ0lYW3qRvuNyMf2aBkS/jD+jOPlbnXrr6NA3epTYIIaTxYODvM/bi3rTKdA7dORjP0vcDNjvPRAojcc16fujsyfj4sfNx4qJux/EjCV0aZGb8E7aMf8TDjD+gF/h6UtyreVuYPKk9nFPcy8DfbXKlPqNW19xsPX4sx/KzdziO6SUW9prnSWhpo2uvs8Yf0J19zP9/sSQDf0IIIc0HA3+fMX387bIcwOzamx34d4RtUp9ECl1GRr8tHMTX330QJndkB0OmFGhgNIlEKu2g8U975uFvEgh4XdzrkdSnI4w9Dq4+5udEqscM/M0Mv9lkzsq626w1TeIpDUOxVHkZfyO7H0+l87r6AHr3Xr1fg0IsSY0/IYSQ5oOBv8+Yrj4TcmQ8Q7HUuMA/FAygLRzQi3vjKXQUCULN7LTZE2BKjsY/qaU9Le4F9ALfRpT6TGoPI5FKW0HnaJx2nm4TNQJr8+7NaMI54LdLfXaP6IuxUq08gUyQrwf+hTL+utTHXBxQ408IIaTZYPrSZ8aSuquPlfGPpayM/L5T2seN1xt9aRiJa1ZGPx/mfrMLsHlHILu41+OMv3iT8U/UIPAHgIGxJNrCQdp5eoAZfMdTGiKhjMzGLrex/wtkJFfF/vbzncf8v+Wk8Y+Gg4jZFnts4EUIIaTZKBo1ichvRKRHRNbYtt0sIiuNn40istLYPk9Exmz7rvVy8o1OUksjqSnLxx/QZTxv9o8CAGZP7Rj3mo6I3uhrLKkVdZgxg9SeITPwNzT+VnGvQsRjqU8o6E3GP2VkiSMhb6Q+5ntlOvuMJjREQwHPeh60IpmAXP97NIP62Ditf0YOZC4KypHhmH/j8WRG6uN0p6stFEAsqVl3GCj18RYROcB2rVgpIoMiconf8yKEkGamlLTZjQB+DuD35gal1HnmYxH5MQB71531Sqklbk2wmbF3g3UK/Oc4BP6d0RB6jOZenUU85SOhACLBgNUMrDMaQiQUQMJs4JXSPJf6BDwr7jXsPAPeZ/wBPSilvt9d7BIcwC71GZ/pN331zW3tZUiu7AuMjKvP+Ne3hYPYPZrIZPwp9fEUpdRrAJYAgIgEAWwFcLuvkyKEkCan6JVNKfUYgH6nfaKbzH8AwE0uz6slML3hdVcf03ozhc39o+iMBLOad5l0RYPoNTL4pQSinfbxkRAiwUB2Ay+PM/7BAKBpjanxB2yBf1yjvt9lrKLbpAallPX/IaPxz2j7rVoL8/9MGdl4c4GRKKLxbzcWFqaLEDv31pRToCeNNvk9EUIIaWaqjZqOB7BTKbXWtm2+iLwgIo+KyPFVHr+pMWULHZEgutoyGf/N/aOYPbUjq3mXSUckZEl3Sgv87eODCAelZg28AKO410M7T8+kPu16PcSe0QQA3RKVgb+72DPxCS2NlCEJiznYeJqLgLFE+UXWGVcfvXA3IEDIQbLVFg4glkxT6uMPHwQTSIQQ4jnVBv7nI/vLejuAOUqpwwBcBuDPIjLR6YUicpGILBeR5b29vVVOozExNc3t4VCW1Gfz7lFHfT+gFzWalp+dJQQ/nZHM+C5D6mN39fG8uDcApD109amd1EdjYa/L2KU+ZkAPwGrqZW/uFcsp/C0nKM9aYKTSiIaCjotqU0pknsupAJi4j4hEAJwD4BaHfS1/nSCEEDep+MomIiEA7wNws7lNKRVXSvUZj1cAWA9gkdPrlVLXKaWWKqWWdnd3Ow1peuzZy/ZwEAHRXX0294856vuBbF1/qVIfk45ICGGb1Efv3Ou9nWfKSztPj+Y/oS0EkezAv1hNBSkPKyBPapaEB7C7+tikPqls3X/FGv9U/r95M/CvZHFBquIsAM8rpXbm7nDrOuH+NxAhhDQm1URNpwJ4VSm1xdwgIt1GkRZEZAGAhQA2VDfF5sVe3Csi6IyGsKl/FGNJDbMdrDz1sZlgvxRLQ/vioMsq7rV37q2BnacHUp+E1bnXG6lPICCY0hHBrmFdJjUST6E9zIy/m9gba9kDf6fGXblSn0o0/vGkhnhKc9T3m/OJJdPWnQbaedaM3DvHruJwc4cQQlqWUuw8bwLwNIADRGSLiHzC2OWkyTwBwGrD3vNWABcrpRwLg0km8Dczi13REF7ZPggAmDMtv9THpBSds2n5GRBdw5xd3Ot9xj8YEE+Ke1NafltGt5gztQOb+nSHJWb83Sef1MfJ1ccq7k1WEvhnu/rkk/C0h4NIaJlFSDl3FUhliEgngNMA/M3vuRBCSCtQNIWplDo/z/YLHbbdBuC26qfVGowlzaZQmcB/fe8wAGD2lHxSn8oy/p2REEQE4WCgpg28gp4V9xoafw/nP396J559Q1+3UuPvPvbGWma9C5DJ7ps2n/o2YzFgLZZL/9yzXX3S1vNczAW4Ke+inaf3KKVGAEzzex6EENIq8MrmIxmpjxGcR0Mw5fD75g38y9P4dxnjzbF2qU+t7Dy9Ke71VuoDAHOndWDbwBhiST0wLaWYmpRORoKTtjL5gD3jbw/8DalPUkN72Lk4N+95wtka/3xSnzZj+26jaRulPoQQQpoNBv4+MpYjKZhgWHp2T4jmlRnYu/WWInfoMAL+DmMBEA4KkimzgVdtMv6eFvd65OoDAPOmdUIp4M3+USPjz0DQTbI0/nGHwN9m52n6748ltbIlOPY7C4U0/mbG37RwZXEvIYSQZoPaBR8ZzfEkN4P6fIW9gF26E0TAwYs8F1MO1GVl/IMYHEtCKaW7+njs4x8UIO2R1CcYkJLeg0qZa9RZvLpjCEBmEUXcIVvqk/kcC2n8xxLpshtrRcpw9QGAPUbGP98CgTQWf12+pfggQghpEXhl85GxpIZwUKysuxnU57Py1MdkS3eK0ZmzqIgEBYlUpllSLYp7U5507lWeNx+bN60TAPDyNr3gmlIfd8kq7jUC+3BQsjr3Bo2FXUbqkypbe28WgMeTpWkQ+FoFAAAgAElEQVT894wlEQ0FPF1UEkIIIX7AwN9HxhJaVvbSlPrka94F2DL+JQb+HdZ4/TxmAy/T2cd7qY94Y+dZA5nS5I4wJraF8LLhtNTO4l5XsTLxyYyTzpSOiCXxiSc1TDYaqWUy/uVLfULBAEIBQULTEE8Wkvro2wdGE5T5EEIIaUoY+PvIaCKV5RRjBueFAv+unEC+GF05C4VwUC/uNTXytcj4e1Hcm0p734NARDBveicz/h4RDAjCQTGkPnpgP7kjnKXxn2QF/pni3o4K+ilEQwHEk2kktDSieYJ6e8afjj6EEEKaEV7dfCS3YLQrqgc5+aw8gWx7zlLIvUMQDgaQTNUy4+9RcW9KIeSx1AcA5k7rtJp4UePvPpFgwPDxT1kdrC07z2QaHdGgLv+xinvTaKtgARYNBzM+/kU0/rtHmPEnhBDSnDDw95Fc2cK0zghEdP/4fFia/bI1/hmpT0JTlqWn11lzL4t7vV60AMA8WyM1ZvzdRw/INYwYi+BoOGjp/WMpDdFQEG2hYJaPf3sF2Xh9gaEVKe7Vtw/GUrTyJIQQ0pQwhekjozka/3OW7I353Z3Ya1Jb3teUq/HPHa937tWsjH/DFvemleeLFkDP+Juwk6v7mBKcVFqhPRJEWziIAcNOM5ZMoy0cQNR2F2A0mSrb1QfQrUN1V58CGn9bsE+pT/OhlCqr/wMhhDQjvLr5yGiOJ3lbOIgj500t+JpwMIBIKGA15irGeDvPAJKasjXAqoHG34uMfypdE6nP/On2jD/XyW4TNRrKjSZS6AiH0B4O2Fx9NLSFgmgLBxC323lWIvUxFhiFXH1y/y+S5sKDryFCCGk4GPj7iKlrLpfj95+Ow2ZPKWls94QoDt57IhbvMwmAbpdY6+JezaMGXrWQ+tgz/h0lLrZI6URDQcvVx8z4m3r+WFJDW3j8tvaKint1SVGiYOdeBv7NDON+Qgih1MdX9OLe8j+CGy48suSxbeEg7vrc8dbzcDAALa0szbTXXvjBQMCTwD9Ro8B/WmcEXdEQhuOpij4rUhhdgqNhzND4Z+n5k2lEwwG0GXcBlFJG597yP/doKIDheMo6Z765mFDq03yklUIQlPoQQlobXt18pBJP8moxM/wjhn2i5xl/gSc+/imtNhp/EbE6+FaiLSeFiYZ07b3pcNVmk/rEU0bG31gMJDUFLa0q+hwioQAGx/TAP9/fTTQUgCkBZ8a/+aDUhxBCGPj7iu5JXuPA3wh6RuKFgyC3CHgo9amFxh/QO/i2h4NWF1niHroER+/c2xEJoc3m6mNab7aF9cDf3F5JUB4NBTAYS+qP87xeRCwZEF19mg9FsQ8hhFDq4xembKESjX81mBl+U/bgtVwm5GHgX6qzUbW865BZmNQRrsm5Wo1oKIA9YwmMxFNGxj+IRCqNdFohZmb8wwH0j6QxZtylqqy4N4jBsaR1znyYfQTo4NR8MONPCCEM/H1D1ywD7TXWjYdzM/4NW9yraqLxB4CzFs/CWYtn1eRcrUY0rLvtjNmKewHd8SqpKbSFdG//WCqT8a9ksRwNByx5W6HAXz9/Mm8dACGEENLI8OrmE6MJPfCuecY/J/CvhZ2nd64+lN40OtGQHtSPJjMafwAYMLLzbeEA2gznHyvjX6HUx37OfJgLD0p9mg9m/AkhhIG/b4xWIVuohrAl9Sme/XSDoIgnxb21svMk3hINBTAcS0FLK0vjDwB7jCZeptSneo1/5jWFsvmWxp/FvU0HNf6EEMLA3zfiRudcrwPvXCJGlrxWGX/vintrJ/Uh3hENBazsfns4k/HfM2rL+BvFvabNZ6WuPtY5C/zdWBl/Sn2aDmb8CSGEgb9vWA20ahy8Zuw8a6Px97K4NxKi1KfRiYaDMP88OqNBK6g3A/+o0bk3Zlh+ApUW99oC/wJBvXl+Wrc2H4z7CSGEgb9vpDT9MlTrrHVuca/XOnlP7TwD/PNtdOwBeXskZFlt7hkzpT66xl9LKwzFMncGyj9P0PFxLmamn1Kf5kMx5U8IIQz8/SJhZPzDNZf6mIG/2bm3UTP+lPo0A/bAv8No1gVkinuj4YzTz27jLkBFGX9blr+4qw+lPs2IB19DhBDScPDq5hOm1KfWzjThHB9/r6VGnhb3UurT8Niz71muPqbGP2TX/et3AWrh6pOvyRdpYBj4E0JI8cBfRH4jIj0issa27ZsislVEVho/77Tt+4qIrBOR10TkDK8m3uj4pvG3SX1CAUHA4260gYBAKSDtcrotqaURptSn4bFn4u0+/vbi3qiV8U9Y48rFXstSqK7FkvrQzrPpoKsPIYSUlvG/EcCZDtuvUkotMX7uBgAROQjABwEcbLzmlyLCK6gDZuAf8qm4dzSheV7YC+hSHwCuZv21tEJa1b4+grhPltQnErKy+buz7DyzpT6VBOXZGv/iUh927m0+KPEnhJASAn+l1GMA+ks83rkA/qKUiiul3gCwDsBRVcyvaUmkzOLeGkt9ghmpTy0CZ/OOgps6f0smRalPwzNe6mMW95quPgG0GYH67pEEoqFARXepSnX1oca/eWHcTwgh1Wn8Pysiqw0p0BRj2z4ANtvGbDG2kRz8tvMcjqdqm/H3IPCv9XtH3CeSlfF30PjnZPwr7XRtD/wL/d2YdxMo9Wk+6OpDCCGVB/7XANgPwBIA2wH8uNwDiMhFIrJcRJb39vZWOI3GJZU2i3trbeeZCcRrETgHxH2pT9KwQg15XJ9AvCdX6tM2zs4zaMlu9owmKvbXN+sEQgEpKK+jnWfzwrCfEEIqDPyVUjuVUppSKg3gemTkPFsBzLYN3dfY5nSM65RSS5VSS7u7uyuZRkOTNKU+NbbzjAYzAU0tZEZBIzh3s7g36ZMVKnEfU+ojYhTyGp+paedp+vgDuu6/rcqMf7FO2ft1d2F6VwSTO8IVnYfUL0z4E0JIhYG/iMyyPX0vANPx504AHxSRqIjMB7AQwLPVTbE5Sfhm55k5Xy2lPikvAn9KfRoeU2/fHg5CRCAiiIYCiCX1z9hu5xlLpivO+Jt/68X+5k89aCaWf+00ZvybELr6EEIIECo2QERuAnASgOkisgXANwCcJCJLoN893QjgUwCglHpJRP4K4GUAKQCfUUpp3ky9sfFLp24PlmtZ3Otuxt+fwmjiPmYG3q7db48EEU+lEQnqhbz2ILxiqY+V8WdA37Iw7ieEkOKBv1LqfIfNNxQY/10A361mUq2AX3aeoYBARL/tzYw/8RszEO+IZL6KdGlPMhOs53j9V3OeQo4+pLlh515CCGHnXt/wK2stIlbAXJOMv7jv6pNIMfBvFpwy/qa0J2pZa7qZ8effTKvyxLpdfk+BEEJ8h1dBn7CCVx+6z0aDtQuCrOJeFyvrzLsHtPNsfCyNf1bgn+2lb7fWrDjjH6bUp9X59eMb/J4CIYT4DiMnn0hqaYQCUlEzomox3XBqkTEPeij1CVHj3/BkpD62zro5mf5wUGD+N6k442+4WdVC3kbqk1iS5WaEEMKroE+k0so3qUrEkvo0qJ0npT5Ng3nXqT2c0fi3W176+r8iYrsLUG3Gn38zrcrGvlG/p0AIIb7Dq6BPJFJp31xpTEvPSA1kD54U96bN+gj++TY6zhr/8d1zzW2VSn0iNZS3EUIIIfUKr4I+kdTSvskOwjXM+HtR3JvJ+FPq0+iEggEEA4LOqC3ID43P7reZC4QKM/6BgCASDFDjTwghpKVh4O8TusbfX6lPoxb30s6zueiKhjCxLdMpt81BllNtxt88Hu08CSGEtDJFffyJNyQ1ldVFt5ZEGr24l1KfpuI3Fx6J2VPbredOen4na89yiYYDdIIihBDS0jDw94mElq6D4t4aZvw9kPowiGsOjpg7Jeu5Gdzbs/PmXYBKXX0A4OIT98NbZk2s+PWEEEJIo8PA3yeSqbRvgasZ8NeixoB2nqRcnDL+pu6/GqnPJ49fUN3ECCGEkAaHKVOf8NPOs6Y+/uJBxp8a/6bGqXFXm0OjL0IIIYSUByMnn0hq/tl5+lHcq7la3MvOvc1Mbude+7ZqpD6EEEJIq8PIySd0H3+fNP5GUXEtG3hR6kNKxbTujDr5+DPwbypEZLKI3Coir4rIKyLyNr/nRAghzQw1/j6R1NLoiPjz9puZ8lpkzD0p7qXUp6kx5TxtTsW9lPo0Gz8DcI9S6l9FJAKgw+8JEUJIM8PIySeSmvKvc6/p6tOwxb2mnScz/s2Io51niBn/ZkNEJgE4AcANAKCUSiil9nh5znU9Q14enpTIhb99Fsd9/yG/p0FIS8LA3yeSftp5+uDj73bGPxwUiDDwb0aioQIaf2b8m4n5AHoB/FZEXhCRX4tIp5cn3DEQ9/LwpEQeea0XW3aP+T0NQloSBv4+kdDSNcm4OxGuZXGveFHc61/XY+I9lqtP2MHVhxn/ZiIE4HAA1yilDgMwAuBy+wARuUhElovI8t7eXj/mSDzkmQ19UC5eGwghxWH05BMpTfnmSuNHxl9zWepDmU/zYjXwshX3nnzADHz4mLnoYMa/mdgCYItS6hnj+a3QFwIWSqnrlFJLlVJLu7u7qz6hAoPMeuK865bhj8s2+T0NQloKFvf6RD3YedayuNfdwD9dk+ZjxB/mTO1AVzSE+dMzqo9DZ0/GobMn+zgr4jZKqR0isllEDlBKvQbgFAAve3tOL49OKuGNXaN+T4GQloKBv0/4qfFv/OJeSn2amb0nt2PNt87wexqkNvwngD8Zjj4bAHzMy5Mx7ieEtDoM/H3CXx//RrfzVAiHKPUhpNFRSq0EsLSG56vVqUgZ3P3idpywqBtdUYYkhHgN06Y+4a+dp37eSA2CZ6+Ke+nhTwgpl9ue3+r3FEgOr+0cxKf/9Dy+fNtqv6dCSEvA6MknWs3O03WNPwN/QkiZ3P/yDr+n0PJ84Nqns54PxzUAwLY9tPckpBYwevIBpRRSaeVf4G8W99ZQ4++2q0+Irj6EENJwPLuxP+v5qs16z7YA+7IQUhOKRn4i8hsR6RGRNbZtPxSRV0VktYjcLiKTje3zRGRMRFYaP9d6OflGxew865czjZNdolcEPMr4U+pDCCmFn563xHpMiX/9wrCfkNpQSvR0I4Azc7bdD+CtSqlDALwO4Cu2feuVUkuMn4vdmWZzkdTSAOCbxv/Ug2bie+9djHnTOjw/V4iBPyHER+rZBjalpfH5v7yA13cO+T2VmrBjIJZ3HxP+hNSGotGTUuoxAP052+5TSqWMp8sA7OvB3JqWTODvT/DaFQ3hgqPnQGrwTRvwpLiXDbwIIaVh7wdRb7y6Ywh3rNyGS/6y0u+p1ISXtg3k3SfM+RNSE9yIPD8O4J+25/NF5AUReVREjs/3olZuxZ4wAv9QC2StrYy/xow/IcRf4qm031NoaSi1IsR/qoqeROQKACkAfzI2bQcwRyl1GIDLAPxZRCY6vdbtVuyNhKXxb4GstVXc63rGn4E/IaQ89pnc7vcUsvj14xv8nkJNKXSTefXWPbWbCCEtTMXRk4hcCOBdAD6kjK4oSqm4UqrPeLwCwHoAi1yYZ1ORTPkr9aklIgIRtxt40c6TEFI+W+vMMvLvK7f5PYWaMndaftlVLFnfd2PGEhruWLmVTeBIw1NRmzwRORPAfwE4USk1atveDaBfKaWJyAIAC6G3YSc2/Nb415pQQJByOfCnnSchhJBa8b27X8Eflm3CaELDUCyJi07Yz+8pEVIRRQN/EbkJwEkApovIFgDfgO7iEwVwv1Egusxw8DkBwLdFJAkgDeBipVS/44FbGFPq0yqBf0DEXalPihp/Qkjz0DqONo2bLd89mgAAfOVvLwIALjh6LrqiFeVOCfGVon+1SqnzHTbfkGfsbQBuq3ZSzY6Z8Y+EWuPbPhQQd4t7fWx+RgghbvPStkG/p1ATGlkl0xlhkE+aA0ZPPtBqUp9AwOWMv5amnSchpKlYubn5i1sbOO5HMOeaQ60/aVRaI/KsMyw7z0BrvP3BgLhb3EupDyGkydjcP1p8UIOTbuBguW84nvX8Q79+Bsde+ZBPsyGkchg9+YBl59lCUh9Xi3sp9SGElMHUzojfUyjKf970gt9T8Jx0fRv3FOTN/mxHqNVbBrB1z1hLLNhIc8HoyQdayc4T0It73cr0KKWQSKVbogcCIcQd+G1RHzRyxj8fx//gYb+nQEhZtEbkWWe0msY/FBCkXCruHU1oAIBOuikQQkpEWsc2h3hEgH9CpElojcizzkimW8zO08Xi3pF4CgADf0JI6XzkbXP9ngJBY2f8Tz9oL7+nQIgrtEbkWWeYUp9W6T7rZnHvsBH40z+ZEFIqHZGg31MgAEq5DIwlNLzjR4/g2TfqqwVQoYz/6i3N78hEmofWiDzrDEvq0yLFvUEXi3tH4pT6EELK4+j50wAA7zt8H59n0tpMbCv+vf3aziFs2DWCD/zqaeweSWBT3wj+9MymGsyuMIWuYOf8/MmazYOQamH05APJVrPzdLG4d9iS+jCDRwgpjRkTowCAI+ZO8XkmOi9uGcDE9ta7/BaSt5q9WcwOuQCwZfcYPnbjs9g1nMB5S2cj5ONd8mKXsB0DMew1qa02kyGkCloj8qwzEqadZwtJfdwq7h2h1IcQUibmvdVaS8x7BmPW45F4Cp//ywt4en0f3v3zJ3DiDx+p7WTqnPOOnA0AeGHTbmtbWinsGtYXAn4XaE+fUNgS9vv3vFqjmRBSHa0RedYZrSj1cSvjP5JgcS8hpEx8+Kp9av0uHPW9B3H3i9sBADc9+ybuWLkN51+/rPaTqRMKXQYmtYcBAEHbnfCXtg1aj/2+WoaK2Po8t7G+ahIIyQcDfx9oNR//YECgsbiXEOIztUz4r9k6AAB43pbBLkYsqWH5xn682decTaFUgU/g14+/MW7MK9sH8w2vO7bsHis+iJA6oDUizzrDtPMslkFoFtwt7mXGnxBSHmLmi13W+qTTyrqD6wZ7RpP412ufxgk/bP6mUF8644Cs5/FUGrGkhp8+sNba9odlmaLeDbtGcNX9r0P5ZAnawE6khGTBwN8HkloakWDAd81irXC3uFd39ekIs7iXEFIaXn3VfvX2F7Hwin+WNLaUr0A3FxH1iP09+NQJC8btv+aR9Xlfe+pPHsXPHlyL/pFE3jGEkOIw8PeBZCptORi0Am4X93ZGggi0yN0SQoh7uJ20/ctzm/OfyziZuegopYnhyT96xIVZ1S/mO3DqW2Y6OvT87MG147bVC0z4k2aBgb8PJLW0r7ZktcbV4t54ijIfQkhZ+OXqA+huNNv2jOHKfxZ3fXFLElnvvOuQWRW/9lv/eNnFmZTPLz90uPV49TdPz9pnSlEJqWdaJ/qsIxKaapnCXsD94l4W9hJCysGUVQ77FJj9xx9X+HLeesMNff6dq7a5MJPyMad++Jwp+PtnjsUvP3Q4JraFs8Yc/I17fZgZIeXROtFnHaFr/FtHquJ24M+MPyGkEn5472ueHLdYQLtqy4An521UGrm8TQRYMnsy3rm48rsWhPgJA38fSGpphEOt89YHRUrSt5aCLvVhYS8hpHS8jjOdvt7MTbe/sLXgaz914vgiVzub+5vH2rNRhEyb+0fHLebyWZG++p0zs57vZvExqXNaJ/qsI1ItKPVxq7h3OK5R6kMIKQuvM8yFvt16h+IVH/eOlVtx/A8exhNrd1V8jHqiESwxn39zN47/wcO46Vnnwu3cP6W2HIe5598svW8DIX7QOtFnHZHQ0i0X+LO4lxDSrFSjXZcC9yNWbt4DAHh1R+M0siqFaq2sz/vV0/i2R0W+r+8YAgA8uS57sVXqR7xsQ5/bUyLEVVon+qwjWk3jH3BR48/AnxBSLoWC61KIJTX85P7XEU9pjvtbxIzHBdx5o555ox+/efINV47ldGwAuOvF7c4DivwpXf+4N/MixC0Y+PtAq9l5hujqQwjxkyrzLP/70Fpc/eBa/OrRDY77Tf33tj1jOPcXT6JvOF5yhjhf8juW1PDKdj3T79b3p59s6B3G9+7WLU3rOe21qW/EcXvjfwKE6JQUfYrIb0SkR0TW2LZNFZH7RWSt8e8UY7uIyNUisk5EVovI4fmP3JokU6q1Gni5VNyb0tKIp9LojDDwJ4SUjj24vu6x/N1h8/GLh/XX/OT+1x33m19vNzzxBlZt3oOL/7gCL28vTZ4zlnC+i/DOqx/Hsg169nltz3CZM64/PvG75Xjo1R4A3tZc3LJ8s7VgqoTn39xTcH+1d48I8ZtS0843AjgzZ9vlAB5USi0E8KDxHADOArDQ+LkIwDXVT7O5aEWNv+ZCce9IXL9A0tWHEFIpZtbZTczAP6mlAQDPbdyNf5ToN58vm7+hN5N5dqtGyk/c+h0mtjknfgZjSVz76Hp86dbVOOtnj7tyrizKnP9oIoWn1/fhthVbsHMw5v58CKmQkqJPpdRjAPpzNp8L4HfG498BeI9t+++VzjIAk0WEhrc2dI1/iwX+LnzpDyf05juU+hBCysFzO09DCOKVo8vfni9sCdoISNbjyj+Ro+ZPddz+rTtfzuqOPBRLVnyOQpR6t+I//vg8zr9+Gb5wyyp85IZnPZkLIZVQTfQ5UyllVr/sADDTeLwPALsP1hZjGzFoNTtPvbi3+uOY7dBZ3EsIKYdqXWSKYeY1QoHyv9eDgeJzO3CvCWUft97Y2JfpR+DFx5Eb6CdSlV105k/vdNxeburq0dd7rcev7RyqaC6EeIEr0afSvczK+n8hIheJyHIRWd7b21v8BU1EqzXw0ot7q4/8h+PM+BNC3CedVnhq/S7Mu/wu/HHZJlz/mHMRb97XG5G/ab9ZDsGA4L5LTyg4ZkG3czDqBVc/uLai36MSbvuPt4/bdv1HlhZ5lbeLuNyF2J7RBFZt3mMt7ko5+/fvGS8n27pnzIXZEVI91USfO00Jj/Fvj7F9K4DZtnH7GtuyUEpdp5RaqpRa2t3dXcU0Gg9d4986BUIBccfVhxl/QkglOH3bxlMa5l1+F07+0SP48m2rccH1zwAAvvb3Nfju3a9Y497Y5ezyYqeabzcBsGhm/WT0f3L/63jPL5709Bzm53HE3CnYeOXZWftOO2gmDijwfmzY5VzonPsZVHqX51+P2BcAsHifSQCAJd++H+f+4kl8486XSj7uNY+MLyA/9sqHsKG38Yu0SeNTTeB/J4CPGo8/CuAO2/aPGO4+xwAYsEmCCIyMfwW3hBuVUEBc8bnOBP4s7iWElI5TrLa5X5eevLFrBLes2DJu/xNrd2HZhj6c9bPHih6/CWpvx5FOK+s7t1rOv25ZWeM/cOTsrOeXnrrIemwvevaCqHE3/sWtA64fe+dg5V2cCXGLUu08bwLwNIADRGSLiHwCwJUAThORtQBONZ4DwN0ANgBYB+B6AJ92fdYNTlJTCIdaJ+MfDAhSrkh9dFcfSn0IIdXw8rZBDMYKB7X/dsMz+OB1yxBLlvDd5XHg78d33vfvfRUHf+NeS2JZDU/ndLMtljQ/59C9rcd//MTRiOVpnPbwaz3W4iS3026lFLsyV3PlTrpR7EZIlZT0baKUOj/PrlMcxioAn6lmUs1OMtVadp6BgMCFuJ9SH0JIReS6yLzzanftHl/ZMYhpnRFXj2nn2P2ne3bsfNxuOAkNx7xomlg4fO6eEMVx+0/HE+t2QUHlvaPysd8+h7MXz8IvPnQ4RvP0Qyh7Zh4Wgv/84XU4YVFrSZtJ/dE60WcdkWgxO8+Qaxl/FvcS0kyIyEYReVFEVorIcu/O49WRdT543TKcdlVxSVC98/cXMuV4Zqz9/l89hdVbalPsa8f8zNIKUAW0VHe96KwkrvQjL3Qu+7wqYetuFvgS/2md6LOOSKVbzM5TpOiXdymMxFMIBsTSYBJCmoKTlVJLlFLF7FxaFvOrc8dADAuvuBtrPNCfA8AlN68ct21z/xjO+bm7xb65wbPpzf/nTx5tbVtg2GpObAt52sDs9he2oMfWYOub/3jZs3MRUg8wgqoxWlpBa7HAP2TYo1Vb4DsST6EzEvTck5sQQvLxlb+txnfvetkqDq4FpsXkw6/1IKkpvOt/n/D8nF4WLOd+g3/5zAMxf3onDp092dr21bPfgt9eeCQOmzOlpLkcvPfEsuexZzSBS29ehaO+9yCA8d7/fcPji3Gdmo+VKvMqpWcDIV7TOtFnnWAW94Rayc7T+LKrVu4zHNco8yGkuVAA7hORFSJykVcncTNXcNOzm3H942/g+B88XNbrDp8zueD+Y/eflndfxLjLubHPW0cbO7scgl6vOGLuFDz8xZOy6reioSBOPnAGAGBCW7jg629bsQVDRYq1nUjYim2VUljXk223ecR/P1DScU4/eK+SxpnXfS2t8K1/vISdgzFs3TOGB17eiXmX34U/LNuEM656DO/75ZNYs3UAj69trR5HpDYwiqoxZuDfShp/M8tRrcx/JJ5iYS8hzcVxSqmtIjIDwP0i8qpSyhLLG4uBiwBgzpw5fs3RFYrdqdy4K/8dBDPjPVxBcFuPlHvXduHMroL7v3DLKodzjB/3xNpd+LcbnsHPLzgM7zpk76w7Cdc9tgHvMBYaBXE47rfPPRifPmm/oovBUcOZbr+v3g0AWNczjMfXZtyI/t/f11iPzbs6uX0OCKmW1ok+64Skpn/TtFIDr5BLGf+RBAN/QpoJpdRW498eALcDOCpnvyuNHp3kGbUm3wzMALVwZ1f9uvGnZ950dU5+Ue6ncdZb98LV5x9W9Xkv/atew/DZP78AAFmNJa99dD1CFSbkwsEAZk/tKDpux2Asy9KT9p7EDxj41xjzP3q4hQpUA+JOxn847oWtHCHED0SkU0QmmI8BnA5gTeFXNS6bqqgJcKMBYiMjIlne/pUSzLkNYC8a3j2atJJUhedS3jlzJV4Lr/hn5vyM+4kPtE70WSeYxUMtVdxr6hpdcPVh115CmoaZAJ4QkVUAngVwl1LqHp/n5Bm9Q5Vr5hu5M1qat3IAACAASURBVPBtDl2R/fJnKBbXe1F7d+Cs/EXHz27sd/18hBSjdaLPOqEVNf5mxr9qqU9co9SHkCZBKbVBKXWo8XOwUuq7fs/JD5z07rm67s/e9Dz+b/W2Wk3JkSO+c3/Zr0mnlaP+vhY4ybty3+vcBVUpi7NylwYXn7Bfma8gxFtaJ/qsE1JpU+PfOm+9W8W9lPoQQiqhHh2A507L1oSff9TsvGOVyujS/aJvJFH2a/Ld5fUt459z2b07p/mX2/0KAGDW5DbXj0lINbRO9FknmFKfVrLzDLpQ3KuUoqsPIaQiStFu15oLjsp2KWpGj/d8jbcqLbY+fuH00gfnnGJdzzA292cKqAdGk3htx1DZcyjkSPSd97x13LaACK750OFln4cQr2AUVWNaUeoTdKG4N55KI5VWzPgTQsqmEZr+tYX8rV9a3ztcfFCZuF2bsNfEyrPnF/1hedbzQ799X7XTGceHj5mLN/tGcP3jb1jbBMD0CdGKj8mEF3Gb1ok+64SMnWfrvPVuFPeOxHX/6s4Ii3sJIY3PWFL3dP/Ls7pF5xfPOAAA8MBlJ5b0+ktvXol4SnNtPss29Ll2LBMtnx1Rheuwr519UMlj+0cSWLszk9HPOxeXuSJnjiLAhioWVT994PVqp0RIFq0TfdYJlp1nC0l9zOJerYqU/4jR+ISZD0JIM9BjFJIOGk252sJBbLzybOw/Q29W9fAXT8JPz1uS9/W3v7AVB3zNPRMkL5yD8kt9KiMaLj1kOflHj+C0qx7DwGhy3FzOLLHTrhPlLiBEBKHc4oIyiCXp+UnchYF/jUm0oI+/qV2tplfJsJHxn9DGwJ8QUr+UWk8QLxLQzZ/eiYP3zm8F6TbbCjYQqwy3k+yV1Gp84Ra9aZc977TvlPaK55Cu4Jea2B6u+HwKDezlSuqS1ok+64RkqgU1/i4U944kDKkPM/6EEB/4wmmLShr3/95VmhzltufH+9v7yaa+yhuM5UPldfWpLOdfSWfdDbtGAAB9I5X3UbDTVULy6QNL9816PnNi5Rr/cu7EvLxtEF+6ZVVFixPSOrRO9FkntKSdpwvFvWbGn4E/IcQPzlqsy0M+dPScguPcrCOuafhWwrzfLHNx4LLEvyI29I7g7f/zYJZkppr3tZRr90kHzMh6HqniDn85c/333y/HLSu2YKsHd29I89A60WedYGr8W8rO08XiXrr6EEL8YP8ZE3DHZ47FN8852HH/hW+fBwA4/aBs/finT6q8gVMpX5lJLY2n1u+q+ByZkxUfsranPPvL7QPuB6DH7j/NevzLEm0ytw3Esp7/6ZlNrs4pl1xFUrQKx6Y/P/Nm2a9pABMr4iMM/GtMohWlPi4U9/YN681jpnREXJkTIYSUy6GzJ+fN+H7znIOx8cqzsdekbMvJHYN60Hn6QTPLPl8p+u4v/HUVLrj+GVx1f+XuL7e/sAV35TSzcpxPmbkb83s7l2oC0wXTu6zHlV5HixXM5vPdP+mA7hLPkP0Lzstp1nbe0vzN2qrBLGD2olCbNA+tE33WCa1o5+lGce+2gTFEggFM62TgTwjxl7fMyi66PWLulLxjew33nkpk16XkSu5ctQ0AcM0j68s/gcGlN68qadxosjz70PnTOx23p6rQoC/eZ5IrxynEaTmLtGVfOQWL95mEL595YEmvz5Xa5NY0LN53EqpFS6txDkNmfPHJ3y13egkhABj415xWtPN0o7h3254Y9prUhkATdrckhNQ37zgwW7N93YePyHo+uYBry96TdAeZRTO7cMvFbyvrvKVk/BcYwfXcnKyyF/QMxooPspEv8zxsWJhWwvtthbNeefPndlFuCwfwj/88btyCLx+vF+kI7EYn6f2+ejf2++rduGfNDgDAv1zzFHYN64vM13aW35GYtA4M/GtMsoXtPKsp7t2+Zwx7T668ayMhhBRj5sQoljpk73PjtGld+p3HDyzdF0tmT8Z/5WSCP3XiAuvxkjmTcevFb8Nlpy3CPpP1RcB+3c6Z8FxKkWyYRcfnHLp3SceshkCZGp18Pv7PbeyveA4iYn1GXmjZH/niSeMy9OXeWRhOFF7YvOewffC99y4GANzw0aXlTTCHi/+4AgCwYtPuqo5DWofWiT7rBNPHv6U0/oHqi3u3D8SszBkhhHiBQHDVeUvwkbfNzdqeq9vviISw6uun43/edwj+/pljccBeE7L2n/qWjFREACydNxWhYAB7T27HrRe/DXd//ngcmPOaSjGD8R9XofH3inyB/6otA1Ud99cfXYrfXngkJlXhj5+PecYdlF9/JBOQR8tM1BVbj7SFg7jg6DnYeOXZOOUtM/HxY+cXXADcsXJr1vMX3mSQTyqn4uhTRA4QkZW2n0ERuUREvikiW23b3+nmhBudVCtq/Kss7tXSCjsGY5jFjD8hxENEgNlTO/Dtc99qbTvpgG587ezx3vyTOsLjJCEm9gZRuVnppfOmIhoKWsc84+D8Rb+l5EqcxsRTWl4P/WooV6GS7/05Yk7+mohSmNwRwck58iu3OfWgmXj9v8/Co186CRPa3F9g2Pn6uw/CKW/J/3fw+b+sxOb+jJXqe3/5lKfzAYCVm/dg3uV3ueMYReqKiqNPpdRrSqklSqklAI4AMArgdmP3VeY+pdTdbky0WRgYSyISDOT9QmxGqi3u7RmKQUsrzGLGnxDiIU5Slvcs2Qdt4fLsGO3f75PanQ0J3r7fNHzulIWW5MOJg0ro3Jt7J3UwlsQBX7sHVz+4rsTZlk65jbfyXeeq8bWvJZFQAHOnlSbLcgO7RCyXZJkX0HRa4T9veqFiCdDT6/sAAI++3lvR60n94tb/vlMArFdKeWuO2wSs2rwHB+9Tuzbs9UAm8K8s8t+2Ry8oM/WxhBDiBfvYMvWPfelkHLNgKk55S/mZ5VAgc2md2O7ceyQQEFx22iJM68rf1bWUBFFul9Z+w0Lzby+43xn4kdd6yhqf76bD0fOnujCb6huB3XPJ8bj+I9Vp7J0otxbC5L/OOBDPfvUUV47ZN5LAP1Ztw79cU+XdAVqDNh1uBf4fBHCT7flnRWS1iPxGRKq7p9dExFMaVm8dcCwea2aqzfibTWAo9SGEeMmv/i3j1jNnWgf+ctHbKpJ52F1+xOM+tfbC0+c29uPm5ZuN87rPq0Xcakrl+IXTXTlOtRy418Rx1p1uUChGz9cjANCvlTMmtuFnH1ziuC8fTu5G1YoK2ASseak68BeRCIBzANxibLoGwH4AlgDYDuDHeV53kYgsF5Hlvb2tcStpzdZBJFLpgp7PzUipxb1DsSSWO7g9bDM8kSn1IYR4yRSX+oQEAoKjXMpqF8Me9L3/2qctP/9yZTmlsGc0Wdb4fF/5bkld46kqrOI85NB9J4/b9uFj9ILxsxbPKvr6c5fsM8461Pw41/cOjxv/o/teG7dtNFFez4VczP4TTPg3H25k/M8C8LxSaicAKKV2KqU0pVQawPUAjnJ6kVLqOqXUUqXU0u7uUrvhNTYrNulB7RFza3NBqBdKLe790zNv4gO/ehoDOReXbXti6IqGMLHN+ZY5IYRUygOXnejJcU05S64jkNvkc84ZqzLwczxmUkP/SALzLr+rJNlPvj4Ebi1KVm7ek3ffc1ec6so5KsHpLsK3zz0Y679XutfJ7z5+ZNZzc7H0PofCXqfmbcf/4OGSz+XEDU+8UdXrSf3iRuB/PmwyHxGxL2ffC2CNC+doCpZv3I250zrQPSG/prMZKVXqs3X3GNIKWL8rO6OxfWAMsya1eZLBIoS0NvtMbsfPLzgM/3v+Ya4e95JTF+HhL56Ut3ttqZh3iK8671DHRUq+JlY7Smy2Va77z5qtuhXnrx8vHhh6YCyUxUg8v1++n9dZpztHIlLWnY4ZE7IXjAERaGmFgbHy7roQkktVgb+IdAI4DcDfbJt/ICIvishqACcDuLSaczQLSik8/+buqm3MGpFSi3t3GheqDb0jWdu3D8Qwi4W9hBAPaI8E8a5D9sa7XW6AFQxI1UE/AHzulIUQAd5x4EzsP6Nr3P58Gf9SGclzZ2B6l7Psqdr8y98/c2x1B7BRr7mgrmgI/3bMHPz2Y0cWH1wiARFc++j4zL7XmN2ASfNQVeCvlBpRSk1TSg3Ytn1YKbVYKXWIUuocpdT26qfZ+GzqG8Wu4QSOmNfKgX/hcTsNTeGGHA3jtj1j2IeFvYSQKthrYmN+h5y4qBtv/M/ZeZtVTazSYz5fxv8TxzlbS/7had28L5+MJ+vYDtsWOixeKiVQJIM+c2JpWf9zl7jf9fi/37MYJx/gXq+Bvy7fjB/eO17L7zV/e35r8UGkoWgMM90mwPTSbbXCXqD04t4eh4x/PKVh13CChb2EkKr44fsPGbetLdz4l0B7s7BKcLoRe+pbZuDiPJ7y9728M+/rcnFaVFRqdelEqEjgHyzxXJ89eX83puMpP3Yo4CWkEhr/W69BWL5pNyZEQ1g0w5027Y2EVdxbIOWfTiv0GBl/u2vBjgF9MTDL4wI5Qkhzc/zC8SYS913iTWFvLalWRu8kFYqEAkVrqirN+Lspzyk1sC/Gwpn1f13OU8pBSNkw8K8RL7y5G4fNnVL01mQzErAy/vnH7BqJQ0srTIiGsKlv1CpY22pYee5NjT8hxGXmTOvwewpV8+KWgeKDClBpjUApgajTod0M/PNN4fA54+00CSmFb9yxBj9/aK3f0/AUBv41om8k0bKdZ0MlFPf2DOrZ/qMXTEVCS2PL7lEAwHajay8Df0IIGc8tK6rr0OskwSxpLVDSesFbqU++RcvPL8jfJIuUz4beYZzy40fQ1wKFvr97ehN+dN/rVR9HKYWklsatK7YgUWf9Jhj414hYQkN7OOj3NHyhlOJe09HnmAXTAGR0/lbXXkp9CCGkarS0ygpEKjUFUlD44i2rcNz3H8o/xuHY7gb+mcf24m0zUVSOBfQXT1+Eb777INfm1qjEkho+8ptns7a948ePYn3vCO56sXW8Wq5+sPKs//WPbcD8r9yNhVf8E1+8ZRV+9mD1Cwk3YeBfI8aSGtojrfl2m1/0hW4p7zQy/m/bTw/8TZ3/toEYpnZG0NaiiyZCCHGTC3/7LBZ97Z/Wc6c+AKUsBp7buBu3rtiCLbvH8o457arHxm1zU+xqdzrab0Z11qmffcdCXHjs/GqnVFM++ra5JY1b1zOEeZffhUO+eS9uf6HwHaIVm3bjsdd7HfcNF+ib0Gz85P7Kg/Xv3v1K1vNbq7wr5zatGYnWmKSWRiqt0BZqzeDVlPqkCoj8dw7GIAIsmjkBkzvC2LBLz/iv6xnG3rTyJIS4zLNXnOL3FHzh8bW7sp47fS+bhbuPfumkst4nLa1w3WPrEUtquP6xDY5j3NT4myVznzxuvufNwuqRjmjxbvYToiGs3KzXgQzGUrj05lUVn68V32M36B2qL4kUA/8aMJbUG6S0R1oz8A+UYOfZMxTDtM4owsEAFkzvxIbeYby2YwjPvtGP0w/aq1ZTJYQ0Md9772IAul1lbmfUViXlUHtl3gSYO62zrPfp7y9sxffufhVXPfA6bnxqo+MYNzuwm/LRYLD1TDOA8f0HnDo7h4KC/pHCgef373kV8y6/q2An5FZg2YY+T45bb45MDPxrQMwI/FtZrhIKCNIF/vp3DsatZisLuruwoXcE1z22Ae3hID58TGm3MwkhpBBhI0CcmKcZViuScCi+ys3R5Ovia/KnZzZh3uV3YfdoAgAwFEtZjmxeYhpGFPPzb1YO3Gui9fhTJyzA/jO6xnV3VgD6R5J5j/HPF7fjmkf0jsCFZFsACl7Dm4EPXrfM7ynUBAb+NSCW0L+cWrW4F9Cz/qmCgX8MM43irAXdnegZiuOOlVtx3pGzMaWz8EWHEEJKwc1ss1+cuKgbJywa35PAiZWb92Bz/2jBMWf+9HHrcea42d/VHyuifb/i9jUAgIGx/AGmF5z51lkAgHMO3ads69CLT9zPo1n5Qzioh3N//vejs7YrBUzuyL/QvebR9dbjtFIFazCaO+z3lnwdsv2AgX8NGGPGH0GRosW9VsZ/up6xUAA+cVxjFVsRQoiX/O7jR+H3Hz8KC3Myu0685xdP4vgfPAylFP78zJvW3ed8LJmt+9/nflXHi7zOxAwa40lnCze3O+TuP6MLG688GwfsNQGL950EIPM7FOKZr56Cy8860NW51JrvvOetWc9PPWgmAGDGhDbMtfWnSCuFSDB/qLfa1gcirQq3ZfvJ/a/ja39/sbIJtwimFXkuPXWk82fgXwMyGv/WfbtDAUEyj59nUkujbyRuaUn369bdGd65eBZmT238BjuEkPrgzLfuheP2n45LT13k91Sq5s//fkzJY/+5Zge+evuLuOqBjFPJW79xLx5fm+3ecqgRPOcGf4Xu1mbx/9u78/ioynt/4J/vTMgeCJAQICxhRzZZUlZB2VeXYm3FWrFq9Xq1Ll2xtpaKrXitXWx76w+X1rZuvVXU4grixYULCrLKIltEMGRhSwJZJpPn98c5M5kks885c85kPu/Xa16ZOefMOd88M3PmO895Fr2K/cVP2o5i8vJtU3DXrEFhxxwpzwg/Fw3M8y4L1HTF30hGdtSrc+D5a8a0+oHj26Jgww+nN69QzUNqh6KU1mwrmH9sOhrWvpLV8lf3+F1+3yu74xxJYMmbicYR2/gDuVkdcOpcg991lTX1UArepj4D8rNx16xB+NHcIfEMkYjauey0FPzjpgntokIh3GQOAP7zmU8AAKdqms/BNfWN+NaTLcdr9zSNaX11NtzEP9DY59+c0Aeje+ciJUjNc6yun1yEJeP74JaL+4fcNlEmhHzhlkkB140o7NTicUqADs4KzQNshNKkFF7fdSLs+CLV6G7Cb97ej+q6+DYJi6dAfVve+rQM9Y3hXTkzGxP/OPDW+Cdx4l+Ym4HjAWpfPGP4e5r6OByCu2YNbhdfzkREZoimP2uo/F30xjqtm/oEulobrkCVPkbKSkvBg4tHIie9uT37kvG9vfcn9e+K9344HXvun2t6LEYpzM1AZojRAD+6dyYeuGIEBuT7b/pVU9+Itz8NL5lf/N8bI44xEmt2luLR9Qfx0Jv7TD2OUZRSuPaJzaisqceeL6twviH0qEfBPpbBhjSPp9CDwFLM6hpY41+Ym4mNhyr9rvPM2lvQkcPrERGFI5rWKsFbcMObtbTeanBBTuQH81F6ti6m50frwcWj8ODiUZYc2yj+fnT59u/olpOOa0OMfNd67oZAwm7SFSXP/1LbENsPyXi5etUmbD5yCsUPrAOgDQP8xNKvRL2/13aW4utf6R16Q5Oxxj8OWOMPFHbOwImquhZTxXuU64l/N73Gn4iIgstJj7zerkSfGDEQT21l6xFILhkS3ihCgWz/4kxMz09mLj+1xH+7cbwFkUSmzuVu06THM6qWnUa4CWbzkVMtHm/5/HTI53RICZxWHw7x+YsXJv5xkOwTeAFaJyWlgBN+an7KqurhdAi6ZjHxJyIKR4co2st/cjR4Au5oB8OdJoMUh/1Tt7Er1mLk8re9j4+dPo8f/E/0swYbYffxsyha9hqORJmAnzmv/ZDZfPgk3thVCgAoPVuLcSvW4lBFDQBg17HAn7HHNhzCR61+TFjB/u+edqBOH9osmZv69NI7Ux0703aoq7KqOuRnp0XUWY2IiIzzz1smYXhPbUKoG0KM2x+pnp3YjNMIDy4eie/PHoz8nNgqyQI1uzXKxkOVON/QsiPrA2v2eu9bUd//ydHT+MlqbSjSd/aWxbSvb6zahFv1DvOv7SzFyXMNmPnIBgChm+Dd/cL2mI5tBCb+cVDHpj4o1Icl89fBt6y6eQx/IiIKTzhj1odrTJ9cdM1OQ8nKhZg+tFuLdRK0y2JoRXlZMT0/mV0xuqf3/pLxffDdmbEPiXrN45tj3kcwu4+fDbreiqY+i/97o3fOAiMn8mu9r1CzSMdjRutQmPjHQW2DGw5pni4+GfXolAER/+Mql1fVIT+HNUJERJH45VdHhN4oTMGa+cR6NTZBmnTb0sT+Xa0OIWKx/lBMFEoprFjTctz+YfpVMztj4h8HtS43Mjo428V08dFKTXGgICfd76/ds7UudA4ypTgREZkr2LdTfk5aTDPdJkIyZFdZaVon7q+N6xXR81Z9a5wZ4QAIXWPvm+oc1tu+t3h+gOedrKlHZY35M9walYntL6tusyzYLMkeGw+a29QqFCb+cVDncid1x16Pws7+x/I/W+tCxwwm/kTJRkScIrJNRNZYHUsi8q1ZfeuuabHtK0Q2dPPU0BNjBXLZhT1Db0R+LRjZAz+YMxjLLxse0fPmDO8edP0T7x+OOqanPiwJur7SZ6K4Ve9px3nTZy6BQL8bxj2wzjt0ppHOnjduwjDfHybzfvd+m/WuMIZEXb+v3LB4osHEPw5qXe6k7tjrUZib0aZzr8vdhPMNbu9060SUVO4EsDfkVuRXVlrz98qQ7rGNtR/qirTDIRga5TECzSpLoTkdgttnDEJ2mrHTLj3wWvQfu1d3fBl0/WMbDnnv+3tbhXq+kXZ8cQYX3v92i2WxNL5Y9OgHQde7m+w/RwET/zioY+IPQKvxLz1TB7fPL+LqOm0mvI5RjElNRIlLRHoBWAjgCatjSVR9u2bh/31rHHYunxOX4/14XuTNfRaO6oFhPdjUJ1k999EX3gFOrPCf+ug7vqpqG1FT34h395WjaNlrOBlB86ITVcEno0uAvD/2xF9ESkRkl4hsF5Et+rIuIrJWRA7ofzvHHmriqm1wJ/WIPh69OmegsUmhvLr5g1NVq12CY1MfoqTzOwA/ApAAX5X2NXd4d3RM186fL982BUvG98Zfvx397KLBtB7tJxzL5g1N6v5tBAz92ZtWh9DCb9d9hom/egeP682d9pa2basfjb9+eAR7SqtCbrf1aOiJwMxkVI3/dKXUaKVUsf54GYB3lFKDALyjP05ans69ya4wt+2QnlX6zH6eLy4iav9EZBGAcqXU1hDb3SwiW0RkS0VFRZyiS1yje+fiwcWjcMmQbugW41jvRvjFZcPRu0um1WGQTX1x6jzufH4bGhrN++3fFKBDQU19o3fd0VNt5xeKxvJ/7wm9EYBtISbSM5tZTX0uB/C0fv9pAFeYdJyEUOdqQjo796KXPpa/75CeVbV6Ux/W+BMlkykALhOREgDPA5ghIv9ovZFSapVSqlgpVZyfnx/vGBPaR/fOsjoELJ1cZHUIFMTqbccsO/Yf1x/AT1bvwivbvzR9QrFATp/TKh49E3vF04GyakvmMwCMSfwVgLdFZKuI3KwvK1BKler3TwAoaP2kZKrJqXO5kZ7C7hQ9PTX+PkN6nvU29WEbf6JkoZS6RynVSylVBOBqAOuVUtdaHBZRuxKqydfdL+yIar+ehLXR3YTNh09GtY9fv/2ZtwmYmelvsNza5dMgf/ZvNnjvDymIraO8r1su9j8a1uzfvhdydCSzGJGNXqSUGgtgPoDbRKTFmGJKe4e0Kfpkqsmp5XCeAIDM1BR0zUptWeOvN/XhqD5ERPY3dVBeWNt5mnaSdS4aGN5rFSlPE5nfv3MA31i1CU9+cCSq/bz3mVbpe66+0bDYWgvU1AcADlec894/UN4830Cw54Rr6qA8PHvTBNw2fWDAbXYes6bJT8yJv1LquP63HMBqAOMBlIlIDwDQ/1o7aKnF2Lm3WWHnjBY1/t7OvWzjT5SUlFL/q5RaZHUcZKwfzRtidQhJL6XVZFKZBlVAevLiA2VasrxizR58fvJckGcEd8zP/D5GiSaF//mlkc2Z4E9aigOTB+YhJ8gwrFZ1eY8p8ReRLBHJ8dwHMAfAbgCvAliqb7YUwCuxHCfRcTjPZoW5GTh2urkjTVWdC06HGHZCIiIi610+utDqEKiV8w3GDKvpb46qMzFMkuU0cdSnaCrvLxqUhxGFsQ1B62nGZMcRrWJtWF0AYLX+j6UAeFYp9aaIfAzgnyJyI4DPAXw9xuMktDpXE5v66HrmZuB/9zf36aiqbUTH9BRbfjiIiIioJX+dUh0xfIdX1xk3s25r0XaglRjr4x1hPN2qvCemxF8pdRjAhX6WnwQwM5Z9txeN7iY0uJuQnsLEHwDyc9JQ63LjXH0jstJSUFXn4og+REQJ5orRPfHy9vjNwEr2se9ENUrPtmyes3ZvGUb26hTV/h5dfxADumUbEVob7igT/13Hz8Z03HCuriRkUx8KrU4fnzYjlUUNAHnZ2tjSlfpMeVW1LrbvJyIy2Q/ntmxzn5ediowOTsy6oAD7VsyLeH+Lx/by3n/2pgkxx0fm+M7Ufqbsd9KD61s8fvSdAzHt78OD5gzp6XZbM2TmibPNE5VG8/kyE7NRk9Xqv/rYuVeTr08qU1GtJf5na10c0YeIyGS3TR+IkpULce+CCwAAC0f2wN4V8/DE0uKI+qD5q0CdPDAPD39tFIb16OjdN9nDHTMHmbZvI1uquE2aw6vRX4eEOHD6tPWxWx9PDp5usjqXlvjb7YW3Sl52KgCfGv+6RnTvlG5lSERE7dLIwk5tmixcO7EvPiurxp2zBse0bxFg9rACrN1TBgC4qrg3riruHdM+yXg5+hV1uzfNcpmU+Ufb1CdWzjAa+dc1GtPZOlJM/E3GxL+l1jX+bOpDRGSOAflZbRL/jFQnHr6qTde8sA3tnoMPDlaiS1YqHr+uONYQKQ4O/WoBHALDE/9A490PKcjB/rLqiPb16o7m2K55fBNG9crFdZP6eif+jFZDo0mXEkIIJ/F/fdcJ7/3zDY1IT3HCEU6v4BixqY/Jal1s6uOrS2YqRICKmgYAYOdeIiKTXD9Fa989sjC6Tpf+/Hj+ULx46yQM72ncPslcToeYMoLMZ2U1fpePiPH9tvHQSTy24RAmr1wfemObCre8dx8/i3P1jRh231v4r7f2mxyVhom/ybxt/DmcJwBtQpEumamoqK5HfaMbda4mdEznhSciinGhiQAAF+lJREFUIqOl6LWHRsxE6tHB6cC4vl0M2x9Z59cxXPkBgCOV0U/alejumBF4Rt5ILPrDByjXW0C89MkxQ/YZChN/k3lG9WFTn2b5OWmorKlHdZ02TTdr/ImIjOepcOqmN7EkAoDFYwpRsnIhvjauF7b9bLah+z5ZU99iks5YNTQ2YfsXZwzbXyhDCnKCrvc0V/5uiE7TrVvsZAWp/J31mw0A/E+MZgZWtZqMo/q0lZetJf5na7VJO9jGn4jIeAPys/H7q0fjksHdrA6FbODwrxZgT2lVi6Y4nbNSDT3GuAfWBVw3d3gB3vq0LKL9PfjGXvzlwxKsvXsaBoVIyo0Qqt/Kv2+/CGdrXejgDF5v3rqhz9afzcbQn73pd1u3nvFHO9lYpFjjb7Lmzr0sao/8nDRUVNejSk/8OZwnEZE5Lh9diE6ZPMcS4HBIzO3vYxHNbLh7vqwCAFToIwGarU/XzKDru2anYkj30D9AWrfxD6fVh5FN8oJhNmoyb+detvH3ystObVnjn8ELT0RERO3Z3bMjH0I2xakl0O4A7WD2llbhT+8ejDqmG6ZENsGZM8xOu9+c0KfNsp3L5wR9zunzrohiiRYTf5OxqU9b+TlpqHM1oVSf2Y5NfYiIiNqP3l0yULJyIcb17QxAm905nJry1pwOLU31nYjr2c1HUbTsNew+fhbzf/8+Ho5hNJz7Lh3mvd83RG0/gLCH2/zqmMI2y8LJdRrNmsnMBxN/k3kmaGDn3mZ52VrnmMMV2lBg7NxLRERkX/5qsFv74EBlm2WePDmcce398YxMdfx0rXfZT1bvAqCNiGOkrxQFH63qL9d/xe/yP10zFpcMyW+xLNrhU2vqG6N6XiSY+JusrsENESAthUXt4Un8D1VoQ4Gxxp+IiMh+Xr9jKp64rhi//OrIkNte++Rm731Pe37P32hbr3t+MPz05d04VFGDbUdP+90ulo6x7/9oOlIcgusnF3mX5aS1bYI8faj/TvILR/XAfYu0KwdpKQ48fcP4qGPZfORU1M8NF7NRk9W63EhPcZoyeUai8gyHdbiiBh2cwo7PREREFvn7jYET1WE9O2LWsAIAwDeKe0e8b0/qE21e7jvz7sxHNqCsqs7vduHsf8fPW7axv/WSAQCA3l0ycfBXC1p0fN7x8zm4//Lh2PrTWQH394clY7w/FvrnZ+MXlw3HBz+egYsH5wd8TtcQoyjFo38vMy6T1brc7NjbiqfG/+ip8+iU0YE/ioiIiCwSbtv7uSMKIt73svlD0S8vCyN7RTea0IbPKlo8rnP5bwO/p7QK5dX+fxQAwF2zBrUZQXBokP/b4RBcN6kIXbMDz4Fx6YU9sfyy4d7HSycXeSs2Awmd15uf+TPxN1mdq4kde1vpkpUKh2iTVbCZDxERkXW6ZIY3lv+MoeEn/p76vDF9OuPdH1yCbL3pzM99OtNG464XtvtdvugPH2D8L98J+LzrJhW1WRan0TNbyA0xtC5r/NuBWpcbaWzK0oLTIeiSpf0qzmHHXiIiIsuk+ExG9cadU009VlqKNRWh/voWW9HM+O83Tgi6Ph6/RZiRmqyuwc0afz88l8M6pnMMfyIiIju4oEdHQ/Zjtwa8rScPu2ZCH8wZ1j2s5942fQAujLKpUmuFuRlB18djEi8m/iardTHx9ycvW7u0yKE8iYiI2peSk+f9LldxqdNuS1plu9+bPTjsMfl/OHcoXrn9IhOiautvGz83/RisbjVZrcvtbdtGzfKzPTX+TPyJiIisNH9Ed6zbWwYAKFm5ENuOnsbwnsbUcttB6xTfblckPLp3Sjf9GMxITVbnakJeNmv8W/M09Wndy56IiIji68/XjmvxeEyfzhZFYg6H3tu4T5dMHD113tLRBPvlZeFI5Tm/6z4u4Tj+Ca/O5easvX54hvTsmMHfnkRERMngyrG9wtpu+32zDT2uJ8//x40TcO+CC9AlxHj6ZgrWjr+8ut704zPxN1ltgxsZHNWnjebOvazxJyIiSgbpHZy4Z/7QFsuuGqf9GHjh5oneZdHOf7TxUKXf5d4a/66Z+M60/lHtOx7cTTbu3CsivUXkXRHZIyKfisid+vLlInJcRLbrtwXGhZt42LnXv+Yafyb+REREieCGKf1i3kfr1PahK0dh7/3z8JWiLt5lvsN+Du8Z/khD1zy+OdbwTDciSN+JeLRAiqUquhHA95VSwwBMBHCbiHhmZvitUmq0fns95igTWJ3LjXTO3NvGsJ4dcWHvXMOGyCIiIiJz3bNgaOiNQmhdq+1wCDJSnd5RdsbrPwB6ddaGvvzXf0yOaP9Pbyxps8zCJv1tPHzVKDzvc3XD1zeKe5t+/KgbWCulSgGU6verRWQvgEKjAmsPmpoU6hubkG7RhBV21iUrFa/cNsXqMIiIiChMHZyxN11udDcn/rdNH9Bi3YfLZqCzPrvta9+dipKT55CR6sSsC7ph3d7ysPZ//5o9WDq5qMWy1uP4WykzNcX746a1ayf2Nf34hjQ+F5EiAGMAeK6x3C4iO0XkKRFpX13DI1DX6AYQfVs1IiIiIjvZuXxOTM93uZu89384t+UVhMLcDGSmanXSnTI74MLeuQCAR74+Ouz9+0vx7VTjD2hXOUpWLmyz3KgJ1IIeO9YdiEg2gBcB3KWUqgLwZwADAIyGdkXgkQDPu1lEtojIloqKiljDsKXaBj3xZxt/IiIiagdiHZQjMy3ynCiSob8bmxRe+uQYVqzZ411ms7w/IGeYk4rFIqbEX0Q6QEv6n1FKvQQASqkypZRbKdUE4HEA4/09Vym1SilVrJQqzs/PjyUM26p1MfEnIiIi8rhlmta8Z8XlwyN6XsnKhShZuRB/vGZMyG0fenMfnvzgiPexleP2200so/oIgCcB7FVK/cZneQ+fzb4KYHf04SW2Opd2OSuNw3kSERFROxNNxaZTb+byrUlFUR1z0aieIbcpq2o5Hn4cKtITRiyzJ00B8C0Au0Rku77sJwCWiMhoaCM2lQC4JaYIE1gda/yJiIionXnzrqnYV1qNiwfn46E39+H5j7+wOqSgEqHG/9NfzI3LcWIZ1ecD+G82ldTDd/o6V98IAJy5l4iIiNqNod07Ymh3rSPqiitGxD3xv3vWYPx23WdxPaaZnv3OBGSlxVIXH76kb4Pyxq5SHCyvMWXfHxyshEOAId1zTNk/ERERkZU6OB3Y/8C8uB5z2uA8AMDDXxsV1+Mabd+KeTjwy/mYPCAvbsdM6sS/pr4R331uGx55e7/h+25qUli97TimDMxDQcd0w/dPREREZAdpKU7sWxG/5H9Mn87YdM9MXFXcGz+/dFjoJ9hUegenIXMjRCKpE/+PjpxEY5PC/x0+iaam1pNIx+bjklM4droWV47tZeh+iYiIiOwm3s2au3fSKlUXM8+KSFIn/h8ePAkAOHPehT2lVYbue/W248hMdWLO8AJD90tERERkR/4mpTJbp4wO+PaUorgfN1EleeJficEF2d77RqlzufHazlLMH9HDOwMdERERERnve7MHWx1CwkjaxL+yph77TlTj8tGFGNgtGx8eOmnYvtfuKUN1fSOuHFto2D6JiIiI7O7FWyfh8euK43rMnBhnE04mSVsdvemwluhPHtAV5VV1+OeWY2hobEJqSuy/hf6940t075iOif27xrwvIiIiokQxrm8XS46bnZaCGn0YdQosaWv8Pzx4EjlpKRhZ2AmTB+ah1uXGtqOnY95vncuN9w9UYvawAjg4VRwRERGR6R5dMtrqEBJC0ib+Gw9VYkL/rkhxOjCxf1c4BIY099l4qBK1LjdmD2OnXiIiIqJ4mDYo3+oQEkJSJv7HTp/H5yfPY8pArSlOp4wOGFnYCRv2l8Plbopp32v3lCM7LQUT+ltzqYuIKBGISLqIfCQiO0TkUxH5hdUxEVHiSonzePiJKilL6ZXtXwIApg1u/nW4aFRP7Dh2FnN++x7e2FUKpSIf17+pSWHd3jJcPDgfaSnxHc+WiCjB1AOYoZS6EMBoAPNEZKLFMRFRO5OdlrTdWf1KusS/zuXGXz4swcWD8zEgP9u7/Kap/fDU9cXo4BTc+swneP7jLyLe987jZ1FRXY9Zw7oZGTIRUbujNDX6ww76zdiZFIkoqf3xmjF4/Y6pVodhK0mX+K/edhyVNfW4ZVr/FstFBDOGFuCNO6dhQr8uWPnGPpysqfe7j6YmhXte2oWlT32EncfOeJev21MGp0MwfQgTfyKiUETEKSLbAZQDWKuU2mx1TETUfiwa1RN9umZaHYatJPT1j7V7yrB+X3nI7RwCzBneHVMH5uHx9w9jRGFHTBrgf6hNp0PwwBUjMP/372PlG/vw8FUXtlivlMLyf3+K5z46iuy0FFz2xw8x64JuyM9Jx/p9ZSju2xm5mamG/H9ERO2ZUsoNYLSI5AJYLSIjlFK7PetF5GYANwNAnz59LIqSiBLFo0vG4I7ntgEA+uVlWRyNPSV04n+4ogbr9paF3K6uwY1nNh/F0O45OFxxDo8uGQORwENtDirIwU1T++OxDYdwYe9cFHRM967bfPgk/vZ/n+Pmaf3x3RkDseq9w3hx6zG4ms7CIcA3J/Y15H8jIkoWSqkzIvIugHkAdvssXwVgFQAUFxezGRARBbVwZA9v4n/fpcMsjsaeEjrxv+XiAbjl4gEht6tvdOOZTUfx6PoDKOqaiQUjuod8zh0zB+L1XaX46cu726y7cmwvLJs3FA6H4PtzhuD7c4ZEFT8RUbISkXwALj3pzwAwG8BDFodFRAnM6RDct2gY7l+zB/26ssbfn4RO/MOVluLEDRf1w9Xje8PlVmEN+ZSZmoLX75yKkspzLZZ3cDowuCA76BUDIiIKqQeAp0XECa2/2T+VUmssjomIEty3pxThijGF6JLFZtf+JEXi75GZGtm/m52WghGFnUyKhogoeSmldgIYY3UcRNS+iAiT/iCSblQfIiIiIqJkxMSfiIiIiCgJMPEnIiIiIkoCTPyJiIiIiJIAE38iIiIioiRgWuIvIvNEZL+IHBSRZWYdh4iIiIiIQjMl8dfHZf4TgPkAhgFYIiKcQo2IiIiIyCJm1fiPB3BQKXVYKdUA4HkAl5t0LCIiIiIiCsGsxL8QwBc+j4/py4iIiIiIyAKWde4VkZtFZIuIbKmoqLAqDCIiIiKipJBi0n6PA+jt87iXvsxLKbUKwCoAEJEKEfk8ymPlAaiM8rl2kMjxM3brJHL8jD1yfS04pq1s3bq1sp19T9gxJsCecdkxJsCecTGm8NkxrmhjCvs7QpRSUew/xE5FUgB8BmAmtIT/YwDXKKU+NeFYW5RSxUbvN14SOX7Gbp1Ejp+xU7zZ8XWzY0yAPeOyY0yAPeNiTOGzY1zxiMmUGn+lVKOI3A7gLQBOAE+ZkfQTEREREVF4zGrqA6XU6wBeN2v/REREREQUvvYwc+8qqwOIUSLHz9itk8jxM3aKNzu+bnaMCbBnXHaMCbBnXIwpfHaMy/SYTGnjT0RERERE9tIeavyJiIiIiCiEhE78RWSeiOwXkYMisszqeIIRkd4i8q6I7BGRT0XkTn15FxFZKyIH9L+drY41EBFxisg2EVmjP+4nIpv18n9BRFKtjjEQEckVkX+JyD4R2SsikxKl7EXkbv09s1tEnhORdDuXvYg8JSLlIrLbZ5nfshbNo/r/sVNExloXecDYH9bfNztFZLWI5Pqsu0ePfb+IzLUmagomnt8TQc7zy0XkuIhs128LfJ7j9z1kZNwiUiIiu/Rjb9GXRfyZFJGl+vYHRGRpDPEM8SmL7SJSJSJ3WVFORp2vApWNiIzTy/6g/lyJMia/5yERKRKRWp8yeyzUsQP9f1HGZdhrJlF8rwWI6QWfeEpEZHs8y0oizPfi9b7yUkol5A3aaEGHAPQHkApgB4BhVscVJN4eAMbq93OgDXc6DMB/AVimL18G4CGrYw3yP3wPwLMA1uiP/wngav3+YwButTrGILE/DeAm/X4qgNxEKHtoM14fAZDhU+bX27nsAUwDMBbAbp9lfssawAIAbwAQABMBbLZh7HMApOj3H/KJfZh+3kkD0E8/HzmtLn/eWryecf2eCHKeXw7gB3629/seMjpuACUA8loti+gzCaALgMP63876/c4GvUYnoI1DHvdyMuJ8FaxsAHykbyv6c+dHGVOg81CR73at9uP32IH+vyjjMuw1QxTfa/5iarX+EQD3xbOsEGG+F6/3leeWyDX+4wEcVEodVko1AHgewOUWxxSQUqpUKfWJfr8awF5oSd3l0JJS6H+vsCbC4ESkF4CFAJ7QHwuAGQD+pW9i59g7QTs5PAkASqkGpdQZJEjZQxt9K0O0+TEyAZTCxmWvlHoPwKlWiwOV9eUA/qY0mwDkikiP+ETalr/YlVJvK6Ua9YeboE1ICGixP6+UqldKHQFwENp5iewjrt8TQc7zgQR6D8Uj7kg/k3MBrFVKnVJKnQawFsA8A+KYCeCQUirY5GymlZNB5yu/ZaOv66iU2qS0bO1vCONcHeF5yK8Qx47quy9AWQUS0WsWbU4RLCZ9n18H8FywfRhdVlHke3F5X3kkcuJfCOALn8fHEPwEaxsiUgRgDIDNAAqUUqX6qhMACiwKK5TfAfgRgCb9cVcAZ3xORHYu/34AKgD8RbSmSk+ISBYSoOyVUscB/BrAUWgJ/1kAW5E4Ze8RqKwT7XN8A7TaFSDxYk9Glr1Grc7zAHC7fhn/KZ/mAoHiMzpuBeBtEdkqIjfryyL9TJpVllejZWJmZTl5GFU2hfp9o+PzPQ8BQD/9u22DiEz1iTXQsY3+7jPiNTMjp5gKoEwpdcBnWVzLKsx8L67vq0RO/BOSiGQDeBHAXUqpKt91+i832w2zJCKLAJQrpbZaHUuUUqBdCvyzUmoMgHPQLrN52bjsO0OrDegHoCeALBhTy2YZu5Z1KCJyL4BGAM9YHQvZm5/z/J8BDAAwGtoP+EfiHNJFSqmxAOYDuE1EpvmutOozqbfhvgzA/+iLrC6nNux2vvJzHioF0Ef/bvsegGdFpGO4+zPg/7Pda+ZjCVr+qIxrWdk130vkxP84gN4+j3vpy2xLRDpAexM8o5R6SV9c5mnaoP8ttyq+IKYAuExESqBdlpsB4PfQLkd5JoGzc/kfA3BMKeWpefsXtB8CiVD2swAcUUpVKKVcAF6C9nokStl7BCrrhPgci8j1ABYB+KZ+wgYSJPYkF/fXyN95XilVppRyK6WaADyO5iZhgeIzNG79yiGUUuUAVuvHj/QzaUZZzgfwiVKqTI/P0nLyYVTZHEfLJjkxxefvPKQ3pTmp398Krf384BDHNuy7z8DX7CQM/F7T97MYwAs+scatrCLM9+L6vkrkxP9jAINE6wWeCu1y4asWxxSQ3tbsSQB7lVK/8Vn1KgBPT+2lAF6Jd2yhKKXuUUr1UkoVQSvn9UqpbwJ4F8DX9M1sGTsAKKVOAPhCRIboi2YC2IMEKHtoTXwmikim/h7yxJ4QZe8jUFm/CuA6fVSDiQDO+lwKtQURmQetmdtlSqnzPqteBXC1iKSJSD8Ag6B1uCL7iOv3RKDzfKt+K18F4BmBJNB7yLC4RSRLRHI896F1Et2NyD+TbwGYIyKd9SuRc/RlsWhRI2tlObViSNno66pEZKL+3rgOUZ6rA52HRCRfRJz6/f7QyuZwiGMb9t1n1Gum/5Ax8nttFoB9Silvk5h4lVUU+V5831cqwh74drpB6wn9GbRfbfdaHU+IWC+CdllnJ4Dt+m0BtHZt7wA4AGAdgC5Wxxri/7gEzaP69If2QT4I7VJtmtXxBYl7NIAtevm/DK2HfEKUPYBfANgH7YT6d2ijJNi27KF9kZcCcEG72nJjoLKGNiLBn/TP8C4AxTaM/SC0dpaez+1jPtvfq8e+HxGMqsBbXF/TuH1PBDnP/11/f++E9iXfI9R7yKi49XPFDv32qWdf0XwmobUtP6jfvh1jWWVBq+Xt5LMs7uVk1PkqUNkAKIZ27j4E4I/QJ06NIia/5yEAV+qv63YAnwC4NNSxA/1/UcZl2GuGKL7X/MWkL/8rgP9otW1cygoR5nvxel95bpy5l4iIiIgoCSRyUx8iIiIiIgoTE38iIiIioiTAxJ+IiIiIKAkw8SciIiIiSgJM/ImIiIiIkgATfyIiIiKiJMDEn4iIiIgoCTDxJyIiIiJKAv8f0FVny0eWgnkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent.train(num_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test\n",
    "\n",
    "Run the trained agent (1 episode)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score:  200.0\n"
     ]
    }
   ],
   "source": [
    "frames = agent.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Render"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Define the Animation class */\n",
       "  function Animation(frames, img_id, slider_id, interval, loop_select_id){\n",
       "    this.img_id = img_id;\n",
       "    this.slider_id = slider_id;\n",
       "    this.loop_select_id = loop_select_id;\n",
       "    this.interval = interval;\n",
       "    this.current_frame = 0;\n",
       "    this.direction = 0;\n",
       "    this.timer = null;\n",
       "    this.frames = new Array(frames.length);\n",
       "\n",
       "    for (var i=0; i<frames.length; i++)\n",
       "    {\n",
       "     this.frames[i] = new Image();\n",
       "     this.frames[i].src = frames[i];\n",
       "    }\n",
       "    document.getElementById(this.slider_id).max = this.frames.length - 1;\n",
       "    this.set_frame(this.current_frame);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.get_loop_state = function(){\n",
       "    var button_group = document[this.loop_select_id].state;\n",
       "    for (var i = 0; i < button_group.length; i++) {\n",
       "        var button = button_group[i];\n",
       "        if (button.checked) {\n",
       "            return button.value;\n",
       "        }\n",
       "    }\n",
       "    return undefined;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.set_frame = function(frame){\n",
       "    this.current_frame = frame;\n",
       "    document.getElementById(this.img_id).src = this.frames[this.current_frame].src;\n",
       "    document.getElementById(this.slider_id).value = this.current_frame;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.next_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.min(this.frames.length - 1, this.current_frame + 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.previous_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.max(0, this.current_frame - 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.first_frame = function()\n",
       "  {\n",
       "    this.set_frame(0);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.last_frame = function()\n",
       "  {\n",
       "    this.set_frame(this.frames.length - 1);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.slower = function()\n",
       "  {\n",
       "    this.interval /= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.faster = function()\n",
       "  {\n",
       "    this.interval *= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_forward = function()\n",
       "  {\n",
       "    this.current_frame += 1;\n",
       "    if(this.current_frame < this.frames.length){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.first_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.last_frame();\n",
       "        this.reverse_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.last_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_reverse = function()\n",
       "  {\n",
       "    this.current_frame -= 1;\n",
       "    if(this.current_frame >= 0){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.last_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.first_frame();\n",
       "        this.play_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.first_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.pause_animation = function()\n",
       "  {\n",
       "    this.direction = 0;\n",
       "    if (this.timer){\n",
       "      clearInterval(this.timer);\n",
       "      this.timer = null;\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.play_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = 1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_forward();}, this.interval);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.reverse_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = -1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_reverse();}, this.interval);\n",
       "  }\n",
       "</script>\n",
       "\n",
       "<div class=\"animation\" align=\"center\">\n",
       "    <img id=\"_anim_imgUJDWEDENQOIUFGBD\">\n",
       "    <br>\n",
       "    <input id=\"_anim_sliderUJDWEDENQOIUFGBD\" type=\"range\" style=\"width:350px\" name=\"points\" min=\"0\" max=\"1\" step=\"1\" value=\"0\" onchange=\"animUJDWEDENQOIUFGBD.set_frame(parseInt(this.value));\"></input>\n",
       "    <br>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.slower()\">&#8211;</button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.first_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.previous_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.reverse_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.pause_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.play_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.next_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.last_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animUJDWEDENQOIUFGBD.faster()\">+</button>\n",
       "  <form action=\"#n\" name=\"_anim_loop_selectUJDWEDENQOIUFGBD\" class=\"anim_control\">\n",
       "    <input type=\"radio\" name=\"state\" value=\"once\" > Once </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"loop\" checked> Loop </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"reflect\" > Reflect </input>\n",
       "  </form>\n",
       "</div>\n",
       "\n",
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Instantiate the Animation class. */\n",
       "  /* The IDs given should match those used in the template above. */\n",
       "  (function() {\n",
       "    var img_id = \"_anim_imgUJDWEDENQOIUFGBD\";\n",
       "    var slider_id = \"_anim_sliderUJDWEDENQOIUFGBD\";\n",
       "    var loop_select_id = \"_anim_loop_selectUJDWEDENQOIUFGBD\";\n",
       "    var frames = new Array(0);\n",
       "    \n",
       "  frames[0] = \"\"\n",
       "  frames[1] = \"\"\n",
       "  frames[2] = \"\"\n",
       "  frames[3] = \"\"\n",
       "  frames[4] = \"\"\n",
       "  frames[5] = \"\"\n",
       "  frames[6] = \"\"\n",
       "  frames[7] = \"\"\n",
       "  frames[8] = \"\"\n",
       "  frames[9] = \"\"\n",
       "  frames[10] = \"\"\n",
       "  frames[11] = \"\"\n",
       "  frames[12] = \"\"\n",
       "  frames[13] = \"\"\n",
       "  frames[14] = \"\"\n",
       "  frames[15] = \"\"\n",
       "  frames[16] = \"\"\n",
       "  frames[17] = \"\"\n",
       "  frames[18] = \"\"\n",
       "  frames[19] = \"\"\n",
       "  frames[20] = \"\"\n",
       "  frames[21] = \"\"\n",
       "  frames[22] = \"\"\n",
       "  frames[23] = \"\"\n",
       "  frames[24] = \"\"\n",
       "  frames[25] = \"\"\n",
       "  frames[26] = \"\"\n",
       "  frames[27] = \"\"\n",
       "  frames[28] = \"\"\n",
       "  frames[29] = \"\"\n",
       "  frames[30] = \"\"\n",
       "  frames[31] = \"\"\n",
       "  frames[32] = \"\"\n",
       "  frames[33] = \"\"\n",
       "  frames[34] = \"\"\n",
       "  frames[35] = \"\"\n",
       "  frames[36] = \"\"\n",
       "  frames[37] = \"\"\n",
       "  frames[38] = \"\"\n",
       "  frames[39] = \"\"\n",
       "  frames[40] = \"\"\n",
       "  frames[41] = \"\"\n",
       "  frames[42] = \"\"\n",
       "  frames[43] = \"\"\n",
       "  frames[44] = \"\"\n",
       "  frames[45] = \"\"\n",
       "  frames[46] = \"\"\n",
       "  frames[47] = \"\"\n",
       "  frames[48] = \"\"\n",
       "  frames[49] = \"\"\n",
       "  frames[50] = \"\"\n",
       "  frames[51] = \"\"\n",
       "  frames[52] = \"\"\n",
       "  frames[53] = \"\"\n",
       "  frames[54] = \"\"\n",
       "  frames[55] = \"\"\n",
       "  frames[56] = \"\"\n",
       "  frames[57] = \"\"\n",
       "  frames[58] = \"\"\n",
       "  frames[59] = \"\"\n",
       "  frames[60] = \"\"\n",
       "  frames[61] = \"\"\n",
       "  frames[62] = \"\"\n",
       "  frames[63] = \"\"\n",
       "  frames[64] = \"\"\n",
       "  frames[65] = \"\"\n",
       "  frames[66] = \"\"\n",
       "  frames[67] = \"\"\n",
       "  frames[68] = \"\"\n",
       "  frames[69] = \"\"\n",
       "  frames[70] = \"\"\n",
       "  frames[71] = \"\"\n",
       "  frames[72] = \"\"\n",
       "  frames[73] = \"\"\n",
       "  frames[74] = \"\"\n",
       "  frames[75] = \"\"\n",
       "  frames[76] = \"\"\n",
       "  frames[77] = \"\"\n",
       "  frames[78] = \"\"\n",
       "  frames[79] = \"\"\n",
       "  frames[80] = \"\"\n",
       "  frames[81] = \"\"\n",
       "  frames[82] = \"\"\n",
       "  frames[83] = \"\"\n",
       "  frames[84] = \"\"\n",
       "  frames[85] = \"\"\n",
       "  frames[86] = \"\"\n",
       "  frames[87] = \"\"\n",
       "  frames[88] = \"\"\n",
       "  frames[89] = \"\"\n",
       "  frames[90] = \"\"\n",
       "  frames[91] = \"\"\n",
       "  frames[92] = \"\"\n",
       "  frames[93] = \"\"\n",
       "  frames[94] = \"\"\n",
       "  frames[95] = \"\"\n",
       "  frames[96] = \"\"\n",
       "  frames[97] = \"\"\n",
       "  frames[98] = \"\"\n",
       "  frames[99] = \"\"\n",
       "  frames[100] = \"\"\n",
       "  frames[101] = \"\"\n",
       "  frames[102] = \"\"\n",
       "  frames[103] = \"\"\n",
       "  frames[104] = \"\"\n",
       "  frames[105] = \"\"\n",
       "  frames[106] = \"\"\n",
       "  frames[107] = \"\"\n",
       "  frames[108] = \"\"\n",
       "  frames[109] = \"\"\n",
       "  frames[110] = \"\"\n",
       "  frames[111] = \"\"\n",
       "  frames[112] = \"\"\n",
       "  frames[113] = \"\"\n",
       "  frames[114] = \"\"\n",
       "  frames[115] = \"\"\n",
       "  frames[116] = \"\"\n",
       "  frames[117] = \"\"\n",
       "  frames[118] = \"\"\n",
       "  frames[119] = \"\"\n",
       "  frames[120] = \"\"\n",
       "  frames[121] = \"\"\n",
       "  frames[122] = \"\"\n",
       "  frames[123] = \"\"\n",
       "  frames[124] = \"\"\n",
       "  frames[125] = \"\"\n",
       "  frames[126] = \"\"\n",
       "  frames[127] = \"\"\n",
       "  frames[128] = \"\"\n",
       "  frames[129] = \"\"\n",
       "  frames[130] = \"\"\n",
       "  frames[131] = \"\"\n",
       "  frames[132] = \"\"\n",
       "  frames[133] = \"\"\n",
       "  frames[134] = \"\"\n",
       "  frames[135] = \"\"\n",
       "  frames[136] = \"\"\n",
       "  frames[137] = \"\"\n",
       "  frames[138] = \"\"\n",
       "  frames[139] = \"\"\n",
       "  frames[140] = \"\"\n",
       "  frames[141] = \"\"\n",
       "  frames[142] = \"\"\n",
       "  frames[143] = \"\"\n",
       "  frames[144] = \"\"\n",
       "  frames[145] = \"\"\n",
       "  frames[146] = \"\"\n",
       "  frames[147] = \"\"\n",
       "  frames[148] = \"\"\n",
       "  frames[149] = \"\"\n",
       "  frames[150] = \"\"\n",
       "  frames[151] = \"\"\n",
       "  frames[152] = \"\"\n",
       "  frames[153] = \"\"\n",
       "  frames[154] = \"\"\n",
       "  frames[155] = \"\"\n",
       "  frames[156] = \"\"\n",
       "  frames[157] = \"\"\n",
       "  frames[158] = \"\"\n",
       "  frames[159] = \"\"\n",
       "  frames[160] = \"\"\n",
       "  frames[161] = \"\"\n",
       "  frames[162] = \"\"\n",
       "  frames[163] = \"\"\n",
       "  frames[164] = \"\"\n",
       "  frames[165] = \"\"\n",
       "  frames[166] = \"\"\n",
       "  frames[167] = \"\"\n",
       "  frames[168] = \"\"\n",
       "  frames[169] = \"\"\n",
       "  frames[170] = \"\"\n",
       "  frames[171] = \"\"\n",
       "  frames[172] = \"\"\n",
       "  frames[173] = \"\"\n",
       "  frames[174] = \"\"\n",
       "  frames[175] = \"\"\n",
       "  frames[176] = \"\"\n",
       "  frames[177] = \"\"\n",
       "  frames[178] = \"\"\n",
       "  frames[179] = \"\"\n",
       "  frames[180] = \"\"\n",
       "  frames[181] = \"\"\n",
       "  frames[182] = \"\"\n",
       "  frames[183] = \"\"\n",
       "  frames[184] = \"\"\n",
       "  frames[185] = \"\"\n",
       "  frames[186] = \"\"\n",
       "  frames[187] = \"\"\n",
       "  frames[188] = \"\"\n",
       "  frames[189] = \"\"\n",
       "  frames[190] = \"\"\n",
       "  frames[191] = \"\"\n",
       "  frames[192] = \"\"\n",
       "  frames[193] = \"\"\n",
       "  frames[194] = \"\"\n",
       "  frames[195] = \"\"\n",
       "  frames[196] = \"\"\n",
       "  frames[197] = \"\"\n",
       "  frames[198] = \"\"\n",
       "  frames[199] = \"\"\n",
       "\n",
       "\n",
       "    /* set a timeout to make sure all the above elements are created before\n",
       "       the object is initialized. */\n",
       "    setTimeout(function() {\n",
       "        animUJDWEDENQOIUFGBD = new Animation(frames, img_id, slider_id, 50, loop_select_id);\n",
       "    }, 0);\n",
       "  })()\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if IN_COLAB:  # for colab\n",
    "    import base64\n",
    "    import glob\n",
    "    import io\n",
    "    import os\n",
    "\n",
    "    from IPython.display import HTML, display\n",
    "\n",
    "\n",
    "    def ipython_show_video(path: str) -> None:\n",
    "        \"\"\"Show a video at `path` within IPython Notebook.\"\"\"\n",
    "        if not os.path.isfile(path):\n",
    "            raise NameError(\"Cannot access: {}\".format(path))\n",
    "\n",
    "        video = io.open(path, \"r+b\").read()\n",
    "        encoded = base64.b64encode(video)\n",
    "\n",
    "        display(HTML(\n",
    "            data=\"\"\"\n",
    "            <video alt=\"test\" controls>\n",
    "            <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
    "            </video>\n",
    "            \"\"\".format(encoded.decode(\"ascii\"))\n",
    "        ))\n",
    "\n",
    "    list_of_files = glob.glob(\"videos/*.mp4\")\n",
    "    latest_file = max(list_of_files, key=os.path.getctime)\n",
    "    print(latest_file)\n",
    "    ipython_show_video(latest_file)\n",
    "\n",
    "else:  # for jupyter\n",
    "    from matplotlib import animation\n",
    "    from JSAnimation.IPython_display import display_animation\n",
    "    from IPython.display import display\n",
    "\n",
    "\n",
    "    def display_frames_as_gif(frames):\n",
    "        \"\"\"Displays a list of frames as a gif, with controls.\"\"\"\n",
    "        patch = plt.imshow(frames[0])\n",
    "        plt.axis('off')\n",
    "\n",
    "        def animate(i):\n",
    "            patch.set_data(frames[i])\n",
    "\n",
    "        anim = animation.FuncAnimation(\n",
    "            plt.gcf(), animate, frames = len(frames), interval=50\n",
    "        )\n",
    "        display(display_animation(anim, default_mode='loop'))\n",
    "\n",
    "\n",
    "    # display \n",
    "    display_frames_as_gif(frames)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
